{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 多臂老虎机问题 使用gym框架,就是一个计算均值的问题，或者期望的问题\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "class Bandit():\n",
    "    def __init__(self,n_bandits,eps=0.2):\n",
    "        self.n_bandits = n_bandits\n",
    "        self.eps = eps\n",
    "        self.probs = np.random.rand(n_bandits)\n",
    "        self.bandits = np.zeros((n_bandits,2)) # 使用一个tuple定义老虎机的奖励值（）均值与动作的次数\n",
    "        self.best_bandit = np.random.randint(n_bandits)\n",
    "        self.best_value = self.bandits[self.best_bandit,1]\n",
    "\n",
    "    def get_reward(self,prob):\n",
    "        reward = 0\n",
    "        for i in range(self.n_bandits):\n",
    "            if np.random.rand() < prob:\n",
    "                reward += 1\n",
    "        return reward\n",
    "\n",
    "    def step(self,reword,action):\n",
    "        new_r = self.bandits[action,1]*self.bandits[action,0]+reword\n",
    "        self.bandits[action,0] += 1\n",
    "        self.bandits[action,1] = new_r/self.bandits[action,0]\n",
    "\n",
    "    def get_best_bandit(self):\n",
    "        return np.argmax(self.bandits[:,1],axis=0)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "bandit = Bandit(10)\n",
    "rewards = [0]\n",
    "for i in  range(500):\n",
    "    if np.random.rand() > bandit.eps:\n",
    "        # 选择最优的动作\n",
    "        action  = bandit.get_best_bandit()\n",
    "    else:\n",
    "        # 随机的选择一个动作\n",
    "        action = np.random.randint(bandit.n_bandits)\n",
    "\n",
    "    reward = bandit.get_reward(bandit.probs[action])\n",
    "    bandit.step(reward,action)\n",
    "    mean_rewards = ((i+1)*rewards[-1]+reward)/(i+2)\n",
    "    rewards.append(mean_rewards)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAGsCAYAAADAPO4TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+rklEQVR4nO3deXxU5b0/8M/sySSZ7CvZ2COBoLIZd0qCRURte71WaEux17pAldL2VWmvCu212OVHtdZLa63YW4to3WtFCCggyhICYSckbAkhO8nMJJNMTmae3x+TGRiSQCY5k8mc+bxfr7zInDlz5pkvox+e5zznOSohhAAREREphjrQDSAiIiJ5MdyJiIgUhuFORESkMAx3IiIihWG4ExERKQzDnYiISGEY7kRERAqjHeo3dDqdOH/+PKKioqBSqYb67YmIiIKWEAJWqxVpaWlQq/vunw95uJ8/fx4ZGRlD/bZERESKUVVVhfT09D6fH/Jwj4qKAuBqmMlkkuWYkiRh06ZNmD17NnQ6nSzHDGWsp/xYU3mxnvJjTeXlr3paLBZkZGR4srQvQx7u7qF4k8kka7gbjUaYTCZ+KWXAesqPNZUX6yk/1lRe/q7n1U5rc0IdERGRwjDciYiIFIbhTkREpDAMdyIiIoVhuBMRESkMw52IiEhhGO5EREQKw3AnIiJSGIY7ERGRwjDciYiIFIbhTkREpDAMdyIiIoVhuBMRESnMkN8VjoiISCmEEKgxd6DeakdWnBHnze1IjDIgQqtCXTvQYLUjLW7o77LHcCciIuqD5HCirNaK0qoWnGxoxYiYcFjaJZxqbMPp7h9bp6PH61QqQAgtHMk1eHTm2CFvt0/h7nA4sGLFCrz++uuora1FWloavvvd7+K///u/r3pvWSIiokBzOAU0aldemdslVNRbUV7Xigu2ThyptiDGqENOqgkVdVYcOGfG0RoLOrucVzymRq1CuE6DVnsXIg1atNq7IARgUAt0SD2Dfyj4FO6//vWvsWbNGvztb39Dbm4u9u7di0WLFiE6OhqPP/64v9pIRETUb+Z2CeeabTjX3I6qC64/y+utOF5jRUu7hHHJUWhstaPBau/X8UxhWuSlxyAr3ojzLe1IiQ7DyIQIjEqIRHZCBDLjjNCqVWiXHIgwaHG+pR3C6UDx9i2YO3O0nz9t73wK9y+//BL33HMP5s6dCwDIzs7GG2+8gT179vT5GrvdDrv9YgEtFgsAQJIkSJI0kDb34D6OXMcLdayn/FhTebGe8hvqmlraJRyrteJcczvOmztQY+5ArbkDuWkmzMpJxPmWDpy9YMPZCzacbGhD5QUbxiRGQqNWofKCDe2SAw6nwI2j4jFjZCyO17Xi6HkLTjfZYO3ouuJ7H6uxeH5PNhkwLikS0eE6jIgJx6FqMzodTkwaEY1JI0zIGxGNzLjwK49OCwccDkCvdtUvMUILSRJQqeSvZ3+PpxJCiP4e9Fe/+hVefvllbNq0CePGjcOBAwcwe/ZsrF69GgsWLOj1NStWrMDKlSt7bF+3bh2MRmN/35qIiIJIRxfQZAcu2FVo6XRtq25Tob7d9bjJ7t9TuZE6gXgDEGcQiDMAiWECaRECKgANHSokGASSw4GwIJt5ZrPZMH/+fJjNZphMpj738yncnU4nfvazn+E3v/kNNBoNHA4Hnn32WSxfvrzP1/TWc8/IyEBjY+MVG+YLSZJQVFSEwsJC6HRDPytRaVhP+bGm8mI95dHe6cDBajNaO7pwfUYUdm7/7Ko1dToF6lvt3UPe7TjX3A57lxPJJgNO1LfiyHkLKi/YYG6/cu8ZANJjwpCdEIG06DCkRoehQ3LigwPn0eUUyIwzIisuHJlxRoxMiECyyYCTDW1Qq1QYkxgBo16DioY2vPrlGUQZdJiYZkJuWhTGJEViREwYjPrApra/vqMWiwUJCQlXDXefPv1bb72Ff/zjH1i3bh1yc3NRWlqKpUuXIi0tDQsXLuz1NQaDAQaDocd2nU4n+3+U/jhmKGM95ceayov17J0QAg6ngFaj9tp2qrENJWebUVrVgoPnWnCsxgqH09W/02lUmBqvxpgpHThc0whrRxduHpuARmsnKuqtqGhoxYnuoe9W+9WDGwBijTqkxxqRFhMGySEwJikSE0dEIz5Cj9w0E2KM+h6vWT53Qp/Hyx/j/XhiRhzuvT6jX20JFLm/o/09lk/h/pOf/ARPPvkkvvnNbwIAJk2ahLNnz2LVqlV9hjsRUagRQqC6pR07yhuxo6IRAsA3rh+BFFM4yuutOHjOjLz0aMzLS4NarYKtsws7yhux92wz7p6chokjor2O53AK2DpdM7FP1LXiWI0F1S3tuCsvFSNiwlHe3WM+dK4F+ypbUGPuQLOtE6YwLVKjwxEfqcfhajOabT3P1yabDDDqtTjd2Iad9WrM/ePOq34+jVqFtJgwZMQakRFrhE1yoLmtE7kjTJicHoPRiZEYERuOSEOQjXkriE+Vt9lsUKu9F7XTaDRwOq98mQAR0XBy6eVQg9Hc1olmWyfqLHbYuxw419yOzcfqUHz6Atouu/b53wdrerz+txvLMCoxErtONXkut3p5+ylMSDXhrsmp0KnV2FHRiF2nmmDvciJcp0H7JZdW/eXzU3A6BSx9TCBrtklegW7QqjE5PQbXZcZgUno0rsuMRVp0GFQqFT45VI3H39gPrVaLnJQoNLTacb6lA1lxRoxOisSYpEiMTYpEblo0RiVGQKfhAqfDmU/hPm/ePDz77LPIzMxEbm4u9u/fj9WrV+PBBx/0V/uISOHKaq14s7gKO0814bHbRyM+Uo9NR+pwtMaCW8Yk4BtT0tHYakdeegwAoENy4Gi1Gd2jyRBCQKVSQQgBc7sErUaN0soWHK0x42R9G8YkReKea9Ow92wzPjp4HqWVLWhotWNUQiSijTrYOrtQcE0y/mNKOradaIC5XcJ38rMRadB6FjCxdnRha1k9TtRZcbzWilijHk4hcLzW2ufn0qpVyE0zYWZOEk42tKHoaC3CdBqMSohArFGP7eUNONfsOmcNAOmx4ai32NHpcOJojQVHL5nR7dYuORCmUyMvPQZ1lg6cbbIBACINWkxIMyE3zYSpWXFIiTYg1qhHU1snSitbYOh+zYRUE/Ta3kN5Vk4Snp3qwLy5sxFmcA2Xy/WPIBp6PoX7iy++iKeeegqPPfYY6uvrkZaWhocffhhPP/20v9pHRApQY25HeV0rmm2d2He2GbERetS0dGB/VTNO1LV69vvBG/u9Xrfn9AX8v6ITAICv5qbgeK1rOFpyCOjUGnxsLsWuUxeg16qh16pRY+7o9f2f/fhYj21ldReD+XC1Bc9vLvc8frO4ClKXE42tneh09ByZvPR9DFo1Ig1axEXokRBpwM1jE/CVnCSMSYq8Yu92a1k9Xt9ViSlZsSi4xrW/U7g+8x8/K8eJulZMyYzFdZkxmJmThEiDFtUt7Zg0IhphOg0utHXio4PnkZsWjWszYnoN4VGJwLTsuD7bcDm9Bl7HYbAHL5/CPSoqCs8//zyef/55PzWHiIKF5HBia1kD4iP1uD4zFgDQ2D2LWqdR4fPyRjS12lFe34ptJxrQ13U5WrUKBdckw2qX8EVFE5KiDLhxdDxM4Tq8v7/aM+T8yZFa7/d3qlB0rN714LK1SNJjw5GXHo3MuAgUHa3FyYY2jIgJx7zJaZg5PhHWji5sOFyLadmxqLfa8fvNrn9AjEuKQlmd1dMjBoAIvQbheg3yRydgxsg4xEe4esQxRh1uGp2AGKNuQCt03j4+CbePT/LaplEB+aPjkT86vtfXpMWEe36Pi9DjO/nZPr8vhQbOdiAin5xsaMWGQzV4fVclai2uHuw916ahyymw8XAtupy9p/i45EhEGLRIjzWiy+HEqMQITE6PwdTsOMRF6NHlcOJUYxvGJEZC3d1jXDEvFzbJgb99eQaHzpkxZ1IKrsuIRWllE/74SSnumjYOeRmxONPYhhijHjeNSUB7pwMZlyw68uPZ41BvtSO1+9yyW8GEZM/vd+WlIsKgRbIpDGW1VuyrbMbYpEgkm1yXaGl5fpmCDMOdiK6qy+HExiN1eGXHKeyvbOnx/Ael5z2/m7pXBZk4IhoTR0QjOlyHORNTMCox8orvodWoMS45ymubWq1CpEGLxTO9r4FKNekgKp2487ZRrkuDxl/5uJf2eHtzadvGp0RhfErUFfYmGv4Y7kTUp4p6Kz4sPY939lWjusU18UunUWFqVhz+c1o67pyUiuLTzXhrbxWSTQbce90I5KZFX+WoRORvDHci8iI5nNhz+gJe3XEaW47Xe7bHRejxrRmZ+FZ+FpKiwjzbbx6bgJvHJgSiqUTUB4Y7EQEAjp634IMD1Xh77zk0tbkWA1epgFk5yZgzMQVz81IRptMEuJVE1B8Md6IQ1thqx4bDtXhjd6XXddWxRh3mTErFf9088qrnyolo+GG4E4UYIQRO1LXitS/P4O2SKkgO1+x2tQq4IzcFd+Wl4Y7cZM4QJwpiDHeiECGEwPbyRvy+6ARKq1o823PTTPjG9em459o0xEf2vMkTEQUfhjuRAjW3deJP209i18kmrLxnItrsXVhddAIlZ5sBAHqtGreOTcTDt43yaQUzIgoODHcihRBCoOhoHT4oPY/Pyuph675xyb0vfeHZx6BV49s3ZOHh20YjMYq9dCKlYrgTBbnOLifW7T6L/9t5Fqca2zzbJ6SaYG6XUN3SDr1GjfkzMvHY7aORZAq7wtGISAkY7kRBqkNyYOORWqwuOuFZC92o1+BbN2Rh7qRU5KVHo8bcgX2VzbhhVDwSeD6dKGQw3ImCzL7KZqz6+BiKzzR7tiVEGrC0YCzuvW4EIg0X/7NOiwm/6tKrRKQ8DHeiILHzZBOe33wCu09f8GxLMYXhm9Mz8NAtoxBh4H/OROTC/xsQDXM15nb86uPj+NcB181ZVCrgP65Px+OzxiI9NnxAtxslImVjuBMNU51dTvx1x2m8+Gk5bJ0OqFXAghlZeOT20RjBoXYiugKGO9EQEkLg/f3V+OuO04gK0+KVhVNxoMqM8norAGD+9Ew0tnbi1S9OY+ORWs9EuSlZsVh5dy4mjuAd14jo6hjuREPkeIsKa/53F47XWj3bJjy90WufbWUN2H36AlrtXQBcE+WWz8nB168fweF3Iuo3hjuRn9RbO7D2izM4Wd/qWmDmmAaAFQatGqnRYTjT3SsP12mQm2bC3rPNnlusTs6IwQPTMnBnXipMYboAfgoiCkYMdyKZWTokrPjgCN7dX93jue/ckIlls8cjKkyHFz8th9MpMH9GFpKiDPj2q7tx5LwFP5o9HgumZ0KtZk+diAaG4U40SEIIvLOvGu/vr8aFtk5UXrB5htUjDVq02rswKiECc5PNeHxuDnQ6V098acE4r+P8/cEZUKnA4XciGjSGO9EA7atsRlmtFW+XnPPckMUtK96I1f95LaZkxcLpFOjqkrBhw4YrHo89dSKSC8OdyEcdkgPPfHAEb+6t8mzTa9T4dn4Wai0dSI4Kw49mj/MsKqNWq9gbJ6IhxXAn8sHJhlYs/sc+z4z3nJQozMxJwqIbs3lDFiIaNhjuRFdQdLQOHZID8yan4cMD57H8nYNo63QgIdKAP3zzWtw4JiHQTSQi6oHhTtSLz8rq8Yct5dhf2QIA+MEb+z3PzRgZhxcfuI49dSIathjuRJf4vLwB7+8/j3f3n4MQ3s9p1So8cttoLC0YC61GHZgGEhH1A8OdQlqH5MDxWismpJrwq4+P4bUvz3iee2B6BhbMyMJP3zmICIMWv/lGHrITIgLXWCKifmK4U8gx2yS8/PlJbC1rQGWTDdbua9Ldvn79CHzrhixcnxkLAPj347cEoplERAPGcKeQYG6X8MnhGpjbJazZehLNNqnHPuE6DX5//2R8dWJqAFpIRCQfhjspWoPVjk1Ha7H2izOoqG/1bB+bFIklXxmDtJhwdDkEdp5qwjenZSCNt1IlIgVguJNi7TrVhIf+by+sHd7D7g/fNgo/mT3ea1Jc/uj4oW4eEZHf+BTu2dnZOHv2bI/tjz32GF566SXZGkU0GNYOCb/dWIa/7zoLIYDxyVGYNzkV903NgEatQkKkIdBNJCLyK5/Cvbi4GA6Hw/P48OHDKCwsxH333Sd7w4gG4kBVCx7+ewlqLR0AgP+Yko7/uXciwnSaALeMiGjo+BTuiYmJXo+fe+45jB49GrfddpusjSIaiH2VzVj41z2w2ruQHW/Es1+bhJu4ghwRhaABn3Pv7OzE66+/jmXLll3xphh2ux12u93z2GKxAAAkSYIk9ZyxPBDu48h1vGAkhEDpOTPGJ0fCqB/cVIrhXM+dp5pwvLYV35yajnD9xd74vsoWPPh/JWizOzAtOxZ/+dZ1iDBoh81nGM41DUasp/xYU3n5q579PZ5KiMvX4eqft956C/Pnz0dlZSXS0tL63G/FihVYuXJlj+3r1q2D0WgcyFtTLzaeU+HjKg2i9QJjTQKVrSp0CUByArelOlE4oudf87k2oLhBDckJRGqBW1OdiNQFoPH9dOiCCq+WqeGECqnhArNGOFHXrsIpiwqnrYATKowxOfH9HCcMHIUnIgWy2WyYP38+zGYzTCZTn/sNONzvuOMO6PV6/Otf/7rifr313DMyMtDY2HjFhvlCkiQUFRWhsLAQOt0wTic/+OJkE17fVYnNxxv63CdCr8GuJ2+HQav2jLLsOXMBj/6jFJZLZpInROrx9Nwc3DQqBju2fnrVeja22nH4vAXjk6NQWtWCM002nGxow+mmNiz/6nhMzYod8Ofq7HJCrYJnRvvmY/V44q2D6Oxy9vmagpxErL4vz6tHP1yE8nfUH1hP+bGm8vJXPS0WCxISEq4a7gMavz179iw2b96Md99996r7GgwGGAw9ZyfrdDrZv0D+OOZwVXK2GUdrLFj54RF0OV3/Pps+Mg4GrRqdXU4YdBqMSojAa1+eQVunA5N+sQWmMC0W3piNMUmRWPpmqWft9Iy4cFQ3t6OxtROPv3kQ908dgRt1V67nthMNePjve9Eh9R62T314DEU/vPWKp2zOt7TD1unAqIQIqNUqdHY58cXJRnxYeh4fHjiPCL0G7y++CV+cbMJT7x8GAMyekIyvTkzBsrcOAAAKrknGzJxEjEmMxPSRccP+vumh9B0dCqyn/FhTecldz/4ea0DhvnbtWiQlJWHu3LkDeTkN0kufVeC3G8s8j8ckReIXd+f2evtRnUaFv3x+GgBg6ejCi59WeJ4rnJCM3903GdHhOnx5shHz/7IbAPDRoVpMv/biMTq7nNBrL14T3thqx4/eKvUK9oRIPWaMjMfn5Q2wdHShor4Vz28uxw8Lx/VoU4utE0vW7ceOikYAQGKUAWE6Nc41t3vdrMXS0YX/+NNONNs6AQAL87Pw87kToNOooNOokRYTjimDGB0gIlIqn8Pd6XRi7dq1WLhwIbRaroEz1E42tOL5zScAuHrqE1JN+GHhOESH9/6vuSVfGQu1SoUpWbHodDjxwuZylNe3IsUUhufvvxYRBtff4Y2jE7D/qUIU/n4bGls7sb9JhWnmDnxxqga/+NdRzMxJxI2jE7C1rB6bj9UDcF0//sGSmwAAeo0aarWr1/znbSexasNxvLClHNNHxuGaVBO0GhUqm2wYnxKFH7zhCnaVyvW6BuvF0zaJUQZMTo9BRlw41n5xBhfaXME+f0YmVtyd6+mZz5vc9zwPIqJQ53M6b968GZWVlXjwwQf90R66ilc+PwXJIXD7+ESs/e60qw5DR4frsPzOazyP50xMxc6TTchOMHqC3S02Qo+vX5+Ol7efwusVGrz+u+2e5z4+VIuPD9V6Hus1avz+/mt7vX784dtG40yTDW/sqcSCV3ZDrQKcl83sCNdp8NbD+RiXEonPTzTC3C5h+sg4pMWEQ6NWwekU2HvGderh/mkZ+OU9E4f9kDsR0XDhc7jPnj0bA5yDR4NUY27Hu/uqAQCLZ44ZUNhp1CrcPLbva7+XFY5DVVMbNhypg06jQoxRj4zYcMQY9VCrgJToMCRFhWFqViwmpPU9meO7N2bjjT2VAHoGOwD87r7JmJQeDQAomJDc43m1WoV/PpKPLqdApIEjREREvuD/NYPIig+PwN7lxLTs2EHNRL+SMJ0Gf/jmZHzw0ce4e+4c6PX6AR1nfEoUfvrVHLy8/SQmpJnwi3smwuEU2HmyCRNHRPfrXDlXlSMiGhiGexDo7HLif7dWYOOROmjVKvzyXv8PUevUGPR7PHr7aDx6+2ivbeOSowZ1TCIiujqG+zBn6ZCwaG0xSs42AwC+d8tI5KTIsz4AEREpE8N9GDPbJHzn1d04cM4Mg1aNBTOy8MOCnpeWERERXYrhPgwJIbC+uAr/u7UCVRfaERehx9+/Nx25adGBbhoREQUBhvswc6axDb/bVIaPDtYAcC0O84//ugHjU3iumoiI+ofhPsz88K1S7K9sAQA8dMtIPHzbaCRE9ly+l4iIqC8M92HkdGObJ9j/8MB1uJursBER0QCor74LDZW/7jgFALh1XCKDnYiIBozhPky8WVyJ13e5VnRbdGN2YBtDRERBjcPyAeZwCvzs3UN4c28VAOCJWWMxMycpwK0iIqJgxnAPoKZWO/6wpdwT7P85NR1LC8YGuFVERBTsGO4BYu2Q8PU1X+Jskw2Aa2b8z+dOCHCriIhICXjOPUB+u7HME+yJUQY8ctvoq7yCiIiof9hzDwCzTcI/954DALz63anIH5WAcD3vgEZERPJguAfAa1+eQbvkwPjkKMwcn+T3O7wREVFo4bD8EKuob8VLn1UAAB6bOZrBTkREsmO4DyGnU+DJdw6i0+HEbVyohoiI/IThPoRe330We882I0KvwbNfm8heOxER+QXDfYi02rvwm0/KAAA/nZOD9FhjgFtERERKxXAfAq32Ljz772NotXdhVEIEvjUjK9BNIiIiBeNseT8rq7XiwdeKUd3SDgD4xpR0qNUcjiciIv9hz92PhBD46TsHPcGeEKnHfVPSA9wqIiJSOvbc/WjjkTqUVrUgXKfBpz++DSmmME6iIyIiv2O4+0mXw4nfbXJNoPuvW0YiNTo8wC0iIqJQwWF5P/nyZBMq6lsRY9ThoVtHBbo5REQUQhjufrLn9AUAwKycZJjCdAFuDRERhRKGu58Un3GF+7Ts2AC3hIiIQg3D3Q+a2zpRWtUCAJiaHRfYxhARUchhuPvBw38vgb3Licw4I0YnRgS6OUREFGIY7jKrt3Zgz5kLUKmAvy6cykvfiIhoyDHcZVZa2QIAGJcUhbHJUYFtDBERhSSfw726uhrf+ta3EB8fj/DwcEyaNAl79+71R9uCkvtc+7UZMQFtBxERhS6fFrFpbm7GTTfdhJkzZ2LDhg1ITExEeXk5YmM5I9xt75lmAMC1mTGBbQgREYUsn8L917/+NTIyMrB27VrPtpEjR8reqGB1sqHVc7795jEJgW4OERGFKJ/C/cMPP8Qdd9yB++67D9u2bcOIESPw2GOP4aGHHurzNXa7HXa73fPYYrEAACRJgiRJA2y2N/dx5DreQAgh8ELRCQDAzHGJSInSBbQ9gzEc6qk0rKm8WE/5saby8lc9+3s8lRBC9PegYWFhAIBly5bhvvvuQ3FxMZ544gn86U9/wsKFC3t9zYoVK7By5coe29etWwej0djftx729jeq8Fq5BioIPJ7rwChToFtERERKY7PZMH/+fJjNZphMfQeNT+Gu1+sxdepUfPnll55tjz/+OIqLi7Fz585eX9Nbzz0jIwONjY1XbJgvJElCUVERCgsLodMFZqnXH6w/gE+O1OG/bs7GT+8YF5A2yGU41FNpWFN5sZ7yY03l5a96WiwWJCQkXDXcfRqWT01NxYQJE7y2XXPNNXjnnXf6fI3BYIDBYOixXafTyf4F8scx+8PpFNjdvZb8nEmpivkPI1D1VDLWVF6sp/xYU3nJXc/+HsunS+FuuukmlJWVeW07ceIEsrKyfDmM4hyvtaLZJsGo1yAvPSbQzSEiohDnU7j/8Ic/xK5du/CrX/0KFRUVWLduHV5++WUsXrzYX+0LCvsqXZe/TcmKhU7DdYGIiCiwfEqiadOm4b333sMbb7yBiRMn4pe//CWef/55LFiwwF/tCwpHa1xXAOSmRQe4JURERD6ecweAu+66C3fddZc/2hK0jnWH+4Q0TpEnIqLA4xjyIDmcAmW1VgDAhFSuJU9ERIHHcB+kA+daYOt0wKBVIzuet3clIqLAY7gPgtMp8MuPjgIA7pyUCi0n0xER0TDANBqEt0vOYX9lCyL0Gjw5JyfQzSEiIgLAcB8wIQRe2FIOAFhaMA7JprAAt4iIiMiF4T5Ah6stqG5ph1GvwbfzQ3sRHyIiGl4Y7gO06WgtAODWsYkI02kC3BoiIqKLGO4D9EVFIwBg1jVJAW4JERGRN4b7AEgOJ46cdy1cMyUrNsCtISIi8sZwH4DyulbYu5yIMmh5bTsREQ07Pi8/G8osHRJ0ajW2lzcAACalR0OtVgW4VURERN4Y7v3U3NaJ2c9vR4PV7tnGIXkiIhqOOCzfT+v2VHoFe05KFL5/66gAtoiIiKh3DPd+EELgzeIqr21LC8YhKkwXoBYRERH1jeHeD+fNHai8YPPalj8qPkCtISIiujKec++HvWcuAADCdRo4nAL5o+MRbWSvnYiIhieGez/sO9sMAPjm9Aw8dMsoRIcz2ImIaPhiuPdDSaUr3KdkxSItJjzArSEiIroynnO/ijZ7F47VWAHw0jciIgoODPerOFDVAodTYERMOFKj2WsnIqLhj+F+FSXd59uvZ6+diIiCBMP9KradcC01Oz2b4U5ERMGB4X4FteYO7O3uuRdOSAlwa4iIiPqH4X4FRUdrAbgm0qVEhwW4NURERP3DcL+C/VUtAIBbxiYEtiFEREQ+YLhfwZFqCwBg0ojoALeEiIio/xjufeiQHKhoaAUATGS4ExFREGG49+F4rRUOp0BCpB5JUYZAN4eIiKjfGO59OFHnWpUuJ8UElUoV4NYQERH1H8O9D1Xdt3jNijcGuCVERES+Ybj3wX3/9sw4hjsREQUXhnsfGO5ERBSsfAr3FStWQKVSef3k5OT4q20B5R6Wz2C4ExFRkPH5fu65ubnYvHnzxQNolXdL+DZ7FxpbOwEAmTznTkREQcbnZNZqtUhJUfY66ye7r2+PMepgCtMFuDVERES+8Tncy8vLkZaWhrCwMOTn52PVqlXIzMzsc3+73Q673e55bLG4Vn2TJAmSJA2gyT25jyPX8XZWuO4Ed216tGzHDCZy15NYU7mxnvJjTeXlr3r293gqIYTo70E3bNiA1tZWjB8/HjU1NVi5ciWqq6tx+PBhREVF9fqaFStWYOXKlT22r1u3Dkbj8Bzy/stxNQ43q3F3pgOzRvS7PERERH5ls9kwf/58mM1mmEymPvfzKdwv19LSgqysLKxevRrf+973et2nt557RkYGGhsbr9gwX0iShKKiIhQWFkKnG9wwutMpMG3VZ7B0dOGdh2cgLz30lp6Vs57kwprKi/WUH2sqL3/V02KxICEh4arhPqjZcDExMRg3bhwqKir63MdgMMBg6Ll8q06nk/0LJMcxz7e0w9LRBa1ahbzMOOg0oXu1oD/+jkIdayov1lN+rKm85K5nf481qORqbW3FyZMnkZqaOpjDDCunG9sAuK5vD+VgJyKi4OVTev34xz/Gtm3bcObMGXz55Zf42te+Bo1GgwceeMBf7Rtyp7rDfWRCRIBbQkRENDA+DcufO3cODzzwAJqampCYmIibb74Zu3btQmJior/aN+TOMNyJiCjI+RTu69ev91c7hg33sPzIRIY7EREFJ55Uvsxp9tyJiCjIMdwvITmcnhvGMNyJiChYMdwvUXXBBodTIFynQXJUWKCbQ0RENCAM90ucaXINyWcnRECtVgW4NURERAPDcL/EqQZXuI/ikDwREQUxhvslOJmOiIiUgOF+iQPnWgAAo5MY7kREFLwY7t0qm2w4XG2BWgXcOlY5i/IQEVHoYbh323ikFgBww6h4xEf2vNENERFRsGC4dyuvtwIAZoyMD3BLiIiIBofh3q3B6rrnfLKJvXYiIgpuIRnuW8vqce9LX+BEndWzraHVFe6JUQx3IiIKbiEZ7t9dW4zSqhYsXV/q2ebuuSdxZToiIgpyIRnubtUt7QAAp1OgsbUTAHvuREQU/EI63J1CAABa2iU4nK7f4yP1gWwSERHRoIV0uHdnu2dIPi5CD50mpEtCREQKEHJJ5u6hAxd77u5wT+T17UREpAAhF+7Ntk7P710OASEE6q0dAHi+nYiIlCHkwr2p9WK4dzqcaOt0oPKCDQCQFsOZ8kREFPxCMNztXo8brfaLt3pNjAxEk4iIiGQVcuHe2Nbp9bih1Y5Tja0AeKtXIiJShpAL98t77mW1Vpzu7rmPTmS4ExFR8Au5cHfPjHd7f3812jod0KhVyIxjuBMRUfALuXCvs7jCfVZOEgBg79lmAEB6bDj02pArBxERKVDIpZn7srfCCckI0138+KnRnClPRETKEHLhXmdxhXt6rBEZsUbPdt4whoiIlCIEw/3ifdtTY8I923kfdyIiUoqQCvcOyQFzuwQASI4OQ9olQ/HJJvbciYhIGUIq3N1D8uE6DaIMWqRGX+y5JzHciYhIIUIs3C8OyatUKqRestxsEteVJyIihQipcPfc/a07yNOiLz3nzp47EREpQ0iFu/t8e3S4HgAQG6HzPMeeOxERKYU20A0YShfD3RXq45OjMDkjBvERekQYQqoURESkYIPquT/33HNQqVRYunSpTM3xr8vDXatR4/3HbsSr350WyGYRERHJasDhXlxcjD//+c/Iy8uTsz1+dXm4A4BKpQpUc4iIiPxiQOHe2tqKBQsW4C9/+QtiY2PlbpPfWDpc4W4K5xA8EREp14BSbvHixZg7dy4KCgrwP//zP1fc1263w26/eCc2i8UCAJAkCZIkDeTte3Af52rHa+m+l3ukXi3beytRf+tJ/ceayov1lB9rKi9/1bO/x/M53NevX499+/ahuLi4X/uvWrUKK1eu7LF906ZNMBqNvbxi4IqKiq74fGWtBoAKJ44cwMfnS2V9byW6Wj3Jd6ypvFhP+bGm8pK7njabrV/7+RTuVVVVeOKJJ1BUVISwsP5dF758+XIsW7bM89hisSAjIwOzZ8+GyWTy5e37JEkSioqKUFhYCJ1O1+d+/6/sc6CtHbNuvgFTsoLndMJQ6289qf9YU3mxnvJjTeXlr3q6R7+vxqdwLykpQX19Pa6//nrPNofDge3bt+OPf/wj7HY7NBqN12sMBgMMhp7XkOt0Otm/QH0d84uKRrz4aTkqL7QDAOKjwvnl7Qd//B2FOtZUXqyn/FhTecldz/4ey6dwnzVrFg4dOuS1bdGiRcjJycFPf/rTHsE+XDz57kFUdQc74D1bnoiISGl8CveoqChMnDjRa1tERATi4+N7bB8uzO2SV7ADgInhTkRECqb45WdP1Fl7bAvTDc8RBiIiIjkM+oLvrVu3ytAM/zle45p8MD45CmV1VuSlRwe4RURERP6l+NVcDpwzAwBmXZOE//vedK4hT0REiqfopPuiohHv7DsHAJg2Mo63dSUiopCgqHPuu05dwIoPj6BDcgAANh2phRDAvMlpuH1cYoBbR0RENDQU1XP/9tq9AID4CD1+MGssWrpvFDM5PZo3iCEiopChqJ6726nGNgBAi63nXeCIiIiUTpHh7hQCADw99xijPpDNISIiGlIKDXfXn2ab6y5wMUb23ImIKHQoM9ydl/XcOSxPREQhRJnhLgScTgFzd7hHs+dOREQhRLHhbu3oQvepd06oIyKikKLIcDe3S8h/bgsAwKjXwKDlWvJERBQ6FHWdu9uuUxc8v/MmMUREFGoU03N3z5C/3IW2zqFtCBERUYApJty7nIFuARER0fCgnHDvo+f+3Ruzh7QdREREgaaccO+l5z41KxZP3TVh6BtDREQUQMoJ91567jmpUdCoecMYIiIKLYoJd0cvPfcIvSIvBiAiIroixYR7bz13I8OdiIhCkHLCvbeeu4HXuBMRUehRTrj30nOPMLDnTkREoUcx4d7rOXeGOxERhSDFhHuX6DkrPkLPYXkiIgo9ygn3XnrunFBHREShSDnh3ss590gOyxMRUQhSTrj31nPnbHkiIgpBygn33mbLc1ieiIhCkGLC3dHrpXDsuRMRUehRTLhzQh0REZGLYtLPHe5fv24EJqVHI8ao401jiIgoJCkm3N3D8nqtGotuGhnYxhAREQWQgoblXb10vVYxH4mIiGhAfErCNWvWIC8vDyaTCSaTCfn5+diwYYO/2uYT92x5vYbhTkREoc2nJExPT8dzzz2HkpIS7N27F1/5yldwzz334MiRI/5qX7+5w13HnjsREYU4n865z5s3z+vxs88+izVr1mDXrl3Izc2VtWG+ck+oY8+diIhC3YAn1DkcDvzzn/9EW1sb8vPz+9zPbrfDbrd7HlssFgCAJEmQJGmgb+9FkiRPuGtUQrbjhip3/VhH+bCm8mI95ceaystf9ezv8VRCiF6Wf+nboUOHkJ+fj46ODkRGRmLdunW48847+9x/xYoVWLlyZY/t69atg9Fo9OWtr+j1CjWKG9S4O9OBWSN8+khERERBwWazYf78+TCbzTCZTH3u53O4d3Z2orKyEmazGW+//TZeeeUVbNu2DRMmTOh1/9567hkZGWhsbLxiw3whSRK+/b9bUNKoxs/mjMeiG7NkOW6okiQJRUVFKCwshE6nC3RzFIE1lRfrKT/WVF7+qqfFYkFCQsJVw93nYXm9Xo8xY8YAAKZMmYLi4mK88MIL+POf/9zr/gaDAQaDocd2nU4n6wd2eibUafjFlIncf0fEmsqN9ZQfayovuevZ32MNevaZ0+n06pkHinv8gavSERFRqPOp5758+XLMmTMHmZmZsFqtWLduHbZu3YqNGzf6q3395l5anuFOREShzqdwr6+vx3e+8x3U1NQgOjoaeXl52LhxIwoLC/3Vvn5zD8trVAx3IiIKbT6F+1//+ld/tWPQ3OGuZs+diIhCnGJWfHGHu5bhTkREIU454d79J8+5ExFRqFNOuLuH5XnOnYiIQpxiwl0IV6hzWJ6IiEKdYsLdwQl1REREABQU7u41dHkpHBERhTrFhLvnOncNw52IiEKb8sKdPXciIgpxygt3nnMnIqIQp5xw7/6T4U5ERKFOOeHOnjsREREABYY7F7EhIqJQp5hwd18Kx0VsiIgo1Ckm3B0cliciIgKgoHAXHJYnIiICoKBw99zylYvYEBFRiFNcuLPnTkREoU454d79J8+5ExFRqFNOuLuH5RnuREQU4hQX7rzlKxERhTrlhHv3n7xxDBERhTrFhLvgde5EREQAFBLuTqeAgCvUGe5ERBTqFBHuDne3HRyWJyIiUkS4O50Xw12tiE9EREQ0cIqIwq5Lwl3LdCciohCniCR0CvbciYiI3BQRhQ7nxd95zp2IiEKdQsL9YrpztjwREYU6ZYS756YxgIo9dyIiCnHKCPfuCXXstRMRETHciYiIFMencF+1ahWmTZuGqKgoJCUl4d5770VZWZm/2tZv7kVsOJmOiIjIx3Dftm0bFi9ejF27dqGoqAiSJGH27Nloa2vzV/v6xb2IDe8IR0REBGh92fmTTz7xevzaa68hKSkJJSUluPXWW2VtmC/ci9jwXu5EREQ+hvvlzGYzACAuLq7Pfex2O+x2u+exxWIBAEiSBEmSBvP2Hp2druOoVZDtmKHMXUPWUj6sqbxYT/mxpvLyVz37ezyVEJcs7+YDp9OJu+++Gy0tLdixY0ef+61YsQIrV67ssX3dunUwGo0DeesezrUBvz2ohUkn8MupDlmOSURENNzYbDbMnz8fZrMZJpOpz/0GHO6PPvooNmzYgB07diA9Pb3P/XrruWdkZKCxsfGKDfPF/rNN+M9XSpBiMuDzn9wmyzFDmSRJKCoqQmFhIXQ6XaCbowisqbxYT/mxpvLyVz0tFgsSEhKuGu4DGpZfsmQJPvroI2zfvv2KwQ4ABoMBBoOhx3adTifbB1apNQBc59z5pZSPnH9H5MKayov1lB9rKi+569nfY/kU7kII/OAHP8B7772HrVu3YuTIkQNqnNzcN4XjbHkiIiIfw33x4sVYt24dPvjgA0RFRaG2thYAEB0djfDwcL80sD+6uteW52x5IiIiH69zX7NmDcxmM26//XakpqZ6ft58801/ta9f3PeNUXMRGyIiIt+H5Ycjzwp17LkTERFxbXkiIiKlYbgTEREpjCLC3bO2PM+5ExERKSPcubY8ERHRRYoId6fgXeGIiIjcFBHunnPuzHYiIiKFhbtaER+HiIhoUBSRhhevcw9wQ4iIiIYBRcShgyvUEREReSgi3N0T6jhbnoiISCHh7r4UjrPliYiIFBLuTs9seYY7ERGRIsKdN44hIiK6SBnhzrXliYiIPBQV7jznTkREpJBwd3JteSIiIg9FhHsX7wpHRETkoYhwd3KFOiIiIg9FxKF7hTquLU9ERKSYcOdd4YiIiNyUEe68zp2IiMhDEeHu5HXuREREHooI9y4uP0tEROShiHB3z5bnIjZEREQKCXcuP0tERHSRssKdw/JEREQKCXfOliciIvJQRLhztjwREdFFigj3LoY7ERGRhyLC3dm9/CyznYiISCHh7j7nzlu+EhERKSXcnbzOnYiIyE1R4c5L4YiIiAYQ7tu3b8e8efOQlpYGlUqF999/3w/N8g0vhSMiIrrI53Bva2vD5MmT8dJLL/mjPQPCFeqIiIgu0vr6gjlz5mDOnDn93t9ut8Nut3seWywWAIAkSZAkyde371WXwzVdXjidsh0zlLlryFrKhzWVF+spP9ZUXv6qZ3+P53O4+2rVqlVYuXJlj+2bNm2C0WiU5T0am9QA1Dh6+BAMtQdlOSYBRUVFgW6C4rCm8mI95ceaykvuetpstn7t5/dwX758OZYtW+Z5bLFYkJGRgdmzZ8NkMsnyHn87txuwmHHttXm4M2+ELMcMZZIkoaioCIWFhdDpdIFujiKwpvJiPeXHmsrLX/V0j35fjd/D3WAwwGAw9Niu0+lk+8Ci+0+9Vr5jkrx/R+TCmsqL9ZQfayovuevZ32Mp41I492x5DSfUERERKSPcPde5B7ghREREw4DPw/Ktra2oqKjwPD59+jRKS0sRFxeHzMxMWRvXXw732vK8FI6IiMj3cN+7dy9mzpzpeeyeLLdw4UK89tprsjXMF+5bvnJteSIiogGE++233w4hxNV3HELuW76qufwsERGRMs65O7n8LBERkYciwp3LzxIREV2krHDnsDwREZFCwp3D8kRERB6KCHcnh+WJiIg8FBHuXRyWJyIi8lBEuLtny3MRGyIiIoWEu3uFOi5iQ0REpJhwd6W7WhGfhoiIaHAUEYeO7gXzeM6diIhIIeHO2fJEREQXKSLcuxjuREREHkEf7u5eO8AbxxAREQEKCHfHJXeoY8+diIhICeHuZLgTERFdSlnhzmF5IiIiBYT7JcPyXKGOiIhICeHuuBjuXKGOiIhICeF+ac+d2U5ERBT84e6+FE4FARXPuRMREQV/uLsXsGGvnYiIyCXow93BcCciIvIS9OHuuZd7gNtBREQ0XAR9JnJYnoiIyFvQh7uT4U5EROQl6MPdfSkcJ8oTERG5BH24d3UvYqMJcDuIiIiGi6APdyd77kRERF6CPtx5KRwREZE3hjsREZHCKCfcA9wOIiKi4SLoM9E9W549dyIiIpcBhftLL72E7OxshIWFYcaMGdizZ4/c7eo3DssTERF58znc33zzTSxbtgzPPPMM9u3bh8mTJ+OOO+5AfX29P9p3VQx3IiIibz6H++rVq/HQQw9h0aJFmDBhAv70pz/BaDTi1Vdf9Uf7rspzKVxA3p2IiGj40fqyc2dnJ0pKSrB8+XLPNrVajYKCAuzcubPX19jtdtjtds9ji8UCAJAkCZIkDaTNl7WpCwCgUUGW49HFOrKe8mFN5cV6yo81lZe/6tnf4/kU7o2NjXA4HEhOTvbanpycjOPHj/f6mlWrVmHlypU9tm/atAlGo9GXt+/VCbMKaUY1EsMEioqKBn08uoj1lB9rKi/WU36sqbzkrqfNZuvXfj6F+0AsX74cy5Yt8zy2WCzIyMjA7NmzYTKZBn38OwEsliQUFRWhsLAQOp1u0McMdRLrKTvWVF6sp/xYU3n5q57u0e+r8SncExISoNFoUFdX57W9rq4OKSkpvb7GYDDAYDD02K7T6WT/AvnjmKGM9ZQfayov1lN+rKm85K5nf4/l04Q6vV6PKVOmYMuWLZ5tTqcTW7ZsQX5+vm8tJCIiIr/weVh+2bJlWLhwIaZOnYrp06fj+eefR1tbGxYtWuSP9hEREZGPfA73+++/Hw0NDXj66adRW1uLa6+9Fp988kmPSXZEREQUGAOaULdkyRIsWbJE7rYQERGRDIJ+bXkiIiLyxnAnIiJSGIY7ERGRwjDciYiIFIbhTkREpDAMdyIiIoVhuBMRESkMw52IiEhhGO5EREQK4/dbvl5OCAGg/7et6w9JkmCz2WCxWHg3IxmwnvJjTeXFesqPNZWXv+rpzk53lvZlyMPdarUCADIyMob6rYmIiBTBarUiOjq6z+dV4mrxLzOn04nz588jKioKKpVKlmNaLBZkZGSgqqoKJpNJlmOGMtZTfqypvFhP+bGm8vJXPYUQsFqtSEtLg1rd95n1Ie+5q9VqpKen++XYJpOJX0oZsZ7yY03lxXrKjzWVlz/qeaUeuxsn1BERESkMw52IiEhhFBHuBoMBzzzzDAwGQ6Cbogisp/xYU3mxnvJjTeUV6HoO+YQ6IiIi8i9F9NyJiIjoIoY7ERGRwjDciYiIFIbhTkREpDAMdyIiIoUJ+nB/6aWXkJ2djbCwMMyYMQN79uwJdJOGre3bt2PevHlIS0uDSqXC+++/7/W8EAJPP/00UlNTER4ejoKCApSXl3vtc+HCBSxYsAAmkwkxMTH43ve+h9bW1iH8FMPHqlWrMG3aNERFRSEpKQn33nsvysrKvPbp6OjA4sWLER8fj8jISHzjG99AXV2d1z6VlZWYO3cujEYjkpKS8JOf/ARdXV1D+VGGhTVr1iAvL8+zold+fj42bNjgeZ61HJznnnsOKpUKS5cu9WxjTX2zYsUKqFQqr5+cnBzP88OqniKIrV+/Xuj1evHqq6+KI0eOiIceekjExMSIurq6QDdtWPr444/Fz3/+c/Huu+8KAOK9997zev65554T0dHR4v333xcHDhwQd999txg5cqRob2/37PPVr35VTJ48WezatUt8/vnnYsyYMeKBBx4Y4k8yPNxxxx1i7dq14vDhw6K0tFTceeedIjMzU7S2tnr2eeSRR0RGRobYsmWL2Lt3r7jhhhvEjTfe6Hm+q6tLTJw4URQUFIj9+/eLjz/+WCQkJIjly5cH4iMF1Icffij+/e9/ixMnToiysjLxs5/9TOh0OnH48GEhBGs5GHv27BHZ2dkiLy9PPPHEE57trKlvnnnmGZGbmytqamo8Pw0NDZ7nh1M9gzrcp0+fLhYvXux57HA4RFpamli1alUAWxUcLg93p9MpUlJSxG9/+1vPtpaWFmEwGMQbb7whhBDi6NGjAoAoLi727LNhwwahUqlEdXX1kLV9uKqvrxcAxLZt24QQrvrpdDrxz3/+07PPsWPHBACxc+dOIYTrH1xqtVrU1tZ69lmzZo0wmUzCbrcP7QcYhmJjY8Urr7zCWg6C1WoVY8eOFUVFReK2227zhDtr6rtnnnlGTJ48udfnhls9g3ZYvrOzEyUlJSgoKPBsU6vVKCgowM6dOwPYsuB0+vRp1NbWetUzOjoaM2bM8NRz586diImJwdSpUz37FBQUQK1WY/fu3UPe5uHGbDYDAOLi4gAAJSUlkCTJq6Y5OTnIzMz0qumkSZOQnJzs2eeOO+6AxWLBkSNHhrD1w4vD4cD69evR1taG/Px81nIQFi9ejLlz53rVDuD3c6DKy8uRlpaGUaNGYcGCBaisrAQw/Oo55HeFk0tjYyMcDodXkQAgOTkZx48fD1CrgldtbS0A9FpP93O1tbVISkryel6r1SIuLs6zT6hyOp1YunQpbrrpJkycOBGAq156vR4xMTFe+15e095q7n4u1Bw6dAj5+fno6OhAZGQk3nvvPUyYMAGlpaWs5QCsX78e+/btQ3FxcY/n+P303YwZM/Daa69h/PjxqKmpwcqVK3HLLbfg8OHDw66eQRvuRMPJ4sWLcfjwYezYsSPQTQlq48ePR2lpKcxmM95++20sXLgQ27ZtC3SzglJVVRWeeOIJFBUVISwsLNDNUYQ5c+Z4fs/Ly8OMGTOQlZWFt956C+Hh4QFsWU9BOyyfkJAAjUbTYyZiXV0dUlJSAtSq4OWu2ZXqmZKSgvr6eq/nu7q6cOHChZCu+ZIlS/DRRx/hs88+Q3p6umd7SkoKOjs70dLS4rX/5TXtrebu50KNXq/HmDFjMGXKFKxatQqTJ0/GCy+8wFoOQElJCerr63H99ddDq9VCq9Vi27Zt+MMf/gCtVovk5GTWdJBiYmIwbtw4VFRUDLvvaNCGu16vx5QpU7BlyxbPNqfTiS1btiA/Pz+ALQtOI0eOREpKilc9LRYLdu/e7alnfn4+WlpaUFJS4tnn008/hdPpxIwZM4a8zYEmhMCSJUvw3nvv4dNPP8XIkSO9np8yZQp0Op1XTcvKylBZWelV00OHDnn9o6moqAgmkwkTJkwYmg8yjDmdTtjtdtZyAGbNmoVDhw6htLTU8zN16lQsWLDA8ztrOjitra04efIkUlNTh993VNbpeUNs/fr1wmAwiNdee00cPXpUfP/73xcxMTFeMxHpIqvVKvbv3y/2798vAIjVq1eL/fv3i7NnzwohXJfCxcTEiA8++EAcPHhQ3HPPPb1eCnfdddeJ3bt3ix07doixY8eG7KVwjz76qIiOjhZbt271ujTGZrN59nnkkUdEZmam+PTTT8XevXtFfn6+yM/P9zzvvjRm9uzZorS0VHzyySciMTExJC81evLJJ8W2bdvE6dOnxcGDB8WTTz4pVCqV2LRpkxCCtZTDpbPlhWBNffWjH/1IbN26VZw+fVp88cUXoqCgQCQkJIj6+nohxPCqZ1CHuxBCvPjiiyIzM1Po9Xoxffp0sWvXrkA3adj67LPPBIAePwsXLhRCuC6He+qpp0RycrIwGAxi1qxZoqyszOsYTU1N4oEHHhCRkZHCZDKJRYsWCavVGoBPE3i91RKAWLt2rWef9vZ28dhjj4nY2FhhNBrF1772NVFTU+N1nDNnzog5c+aI8PBwkZCQIH70ox8JSZKG+NME3oMPPiiysrKEXq8XiYmJYtasWZ5gF4K1lMPl4c6a+ub+++8XqampQq/XixEjRoj7779fVFRUeJ4fTvXk/dyJiIgUJmjPuRMREVHvGO5EREQKw3AnIiJSGIY7ERGRwjDciYiIFIbhTkREpDAMdyIiIoVhuBMRESkMw52IiEhhGO5EREQKw3AnIiJSmP8PkN7X0Bk16pYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 600x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_result(rewards):\n",
    "    plt.figure(figsize=(6,5))\n",
    "    plt.grid()\n",
    "    plt.plot(rewards)\n",
    "\n",
    "plot_result(rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzoAAAGsCAYAAAAVEdLDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAb+0lEQVR4nO3df3SWd33/8ReEJqETaC0SCmZL59HV2hYqkZyUubmzrGwyPD3HzVqrsOjYmYOtbY4egytkrpbQdmXMgWVlRXfO7CnTrUePVDw1itoVDxVkx571x1wPllNPApxOUukxuCT7Y9+l5kug3FB6J58+Hudcf+STz3Xf79znbk+f57rvq5OGh4eHAwAAUJDJ1R4AAADg5SZ0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4U6o9wOkYGhrKj370o0ybNi2TJk2q9jgAAECVDA8P5/nnn8+cOXMyefLJr9tMiND50Y9+lMbGxmqPAQAAjBMHDx7M61//+pP+fkKEzrRp05L87x8zffr0Kk8DAABUS39/fxobG0ca4WQmROj838fVpk+fLnQAAICX/EqLmxEAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRnSrUHAADOvabOHdUe4RVxYP2Sao8AjBOu6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGmVHsAAE5fU+eOao9wzh1Yv6TaIwBQAFd0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOKcUehs3rw5TU1Nqa+vT0tLS/bs2XPK/Rs3bsyv/MqvZOrUqWlsbMzNN9+cn/70p2c0MAAAwEupOHS2b9+ejo6OdHV1Zd++fZk3b14WL16cQ4cOjbn/vvvuS2dnZ7q6uvL444/n3nvvzfbt2/Pxj3/8rIcHAAAYS8Whs2HDhqxYsSLt7e257LLLsmXLlpx//vnZtm3bmPsfeeSRLFq0KO973/vS1NSUa665Jtdff/1LXgUCAAA4UxWFzvHjx7N37960tbW9+ACTJ6etrS27d+8e85yrr746e/fuHQmbp59+Og8++GDe+c53nvR5BgYG0t/fP+oAAAA4XVMq2XzkyJEMDg6moaFh1HpDQ0OeeOKJMc953/velyNHjuRXf/VXMzw8nP/+7//OH//xH5/yo2vd3d35xCc+UcloAAAAI875Xdd27dqVdevW5dOf/nT27duXf/mXf8mOHTty6623nvSc1atX5+jRoyPHwYMHz/WYAABAQSq6ojNz5szU1NSkr69v1HpfX19mz5495jlr1qzJBz7wgfzhH/5hkuSKK67IsWPH8kd/9Ef58z//80yefGJr1dXVpa6urpLRAAAARlR0Rae2tjYLFixIT0/PyNrQ0FB6enrS2to65jkvvPDCCTFTU1OTJBkeHq50XgAAgJdU0RWdJOno6Mjy5cvT3NychQsXZuPGjTl27Fja29uTJMuWLcvcuXPT3d2dJFm6dGk2bNiQq666Ki0tLfnBD36QNWvWZOnSpSPBAwAA8HKqOHSuu+66HD58OGvXrk1vb2/mz5+fnTt3jtyg4Jlnnhl1BeeWW27JpEmTcsstt+TZZ5/N6173uixdujS33Xbby/dXAAAA/JxJwxPg82P9/f2ZMWNGjh49munTp1d7HICqaercUe0RzrkD65dUe4QivRreO4n3D7wanG4bnPO7rgEAALzShA4AAFAcoQMAABRH6AAAAMUROgAAQHEqvr00AADwolfDXQ0n4h0NXdEBAACKI3QAAIDiCB0AAKA4vqMDAMApvRq+g5JMzO+hcHKu6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFmVLtAQAAqq2pc0e1R3hFHFi/pNojwCvmjK7obN68OU1NTamvr09LS0v27Nlzyv0//vGPs3Llylx88cWpq6vLm970pjz44INnNDAAAMBLqfiKzvbt29PR0ZEtW7akpaUlGzduzOLFi/Pkk09m1qxZJ+w/fvx4fuu3fiuzZs3KF77whcydOzc//OEPc8EFF7wc8wMAAJyg4tDZsGFDVqxYkfb29iTJli1bsmPHjmzbti2dnZ0n7N+2bVuee+65PPLIIznvvPOSJE1NTad8joGBgQwMDIz83N/fX+mYAADAq1hFH107fvx49u7dm7a2thcfYPLktLW1Zffu3WOe86UvfSmtra1ZuXJlGhoacvnll2fdunUZHBw86fN0d3dnxowZI0djY2MlYwIAAK9yFYXOkSNHMjg4mIaGhlHrDQ0N6e3tHfOcp59+Ol/4whcyODiYBx98MGvWrMldd92VT37ykyd9ntWrV+fo0aMjx8GDBysZEwAAeJU753ddGxoayqxZs3LPPfekpqYmCxYsyLPPPps777wzXV1dY55TV1eXurq6cz0aAABQqIpCZ+bMmampqUlfX9+o9b6+vsyePXvMcy6++OKcd955qampGVl785vfnN7e3hw/fjy1tbVnMDYAAMDJVfTRtdra2ixYsCA9PT0ja0NDQ+np6Ulra+uY5yxatCg/+MEPMjQ0NLL21FNP5eKLLxY5AADAOVHx/0eno6MjW7duzT/8wz/k8ccfz4c//OEcO3Zs5C5sy5Yty+rVq0f2f/jDH85zzz2XG2+8MU899VR27NiRdevWZeXKlS/fXwEAAPBzKv6OznXXXZfDhw9n7dq16e3tzfz587Nz586RGxQ888wzmTz5xX5qbGzMV7/61dx888258sorM3fu3Nx444352Mc+9vL9FQAAAD/njG5GsGrVqqxatWrM3+3ateuEtdbW1nznO985k6cCAACoWMUfXQMAABjvhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQnCnVHmAiaurcUe0RzrkD65dUewQAADhjrugAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFCcKdUeAF5tmjp3VHuEc+7A+iXVHgEAeJVzRQcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhnFDqbN29OU1NT6uvr09LSkj179pzWeffff38mTZqUa6+99kyeFgAA4LRUHDrbt29PR0dHurq6sm/fvsybNy+LFy/OoUOHTnnegQMH8pGPfCRvf/vbz3hYAACA01Fx6GzYsCErVqxIe3t7LrvssmzZsiXnn39+tm3bdtJzBgcHc8MNN+QTn/hEfvmXf/msBgYAAHgpFYXO8ePHs3fv3rS1tb34AJMnp62tLbt37z7peX/5l3+ZWbNm5UMf+tBpPc/AwED6+/tHHQAAAKerotA5cuRIBgcH09DQMGq9oaEhvb29Y57z8MMP5957783WrVtP+3m6u7szY8aMkaOxsbGSMQEAgFe5c3rXteeffz4f+MAHsnXr1sycOfO0z1u9enWOHj06chw8ePAcTgkAAJRmSiWbZ86cmZqamvT19Y1a7+vry+zZs0/Y/5//+Z85cOBAli5dOrI2NDT0v088ZUqefPLJvOENbzjhvLq6utTV1VUyGgAAwIiKrujU1tZmwYIF6enpGVkbGhpKT09PWltbT9h/6aWX5vvf/372798/crzrXe/Kb/zGb2T//v0+kgYAAJwTFV3RSZKOjo4sX748zc3NWbhwYTZu3Jhjx46lvb09SbJs2bLMnTs33d3dqa+vz+WXXz7q/AsuuCBJTlgHAAB4uVQcOtddd10OHz6ctWvXpre3N/Pnz8/OnTtHblDwzDPPZPLkc/rVHwAAgFOqOHSSZNWqVVm1atWYv9u1a9cpz/3sZz97Jk8JAABw2lx6AQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKM6XaA1Ceps4d1R7hnDuwfkm1RwAA4BRc0QEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIpzRqGzefPmNDU1pb6+Pi0tLdmzZ89J927dujVvf/vbc+GFF+bCCy9MW1vbKfcDAACcrYpDZ/v27eno6EhXV1f27duXefPmZfHixTl06NCY+3ft2pXrr78+3/jGN7J79+40NjbmmmuuybPPPnvWwwMAAIyl4tDZsGFDVqxYkfb29lx22WXZsmVLzj///Gzbtm3M/Z/73OfyJ3/yJ5k/f34uvfTS/P3f/32GhobS09Nz1sMDAACMpaLQOX78ePbu3Zu2trYXH2Dy5LS1tWX37t2n9RgvvPBCfvazn+W1r33tSfcMDAykv79/1AEAAHC6KgqdI0eOZHBwMA0NDaPWGxoa0tvbe1qP8bGPfSxz5swZFUv/v+7u7syYMWPkaGxsrGRMAADgVe4Vveva+vXrc//99+eBBx5IfX39SfetXr06R48eHTkOHjz4Ck4JAABMdFMq2Txz5szU1NSkr69v1HpfX19mz559ynP/6q/+KuvXr8/Xvva1XHnllafcW1dXl7q6ukpGAwAAGFHRFZ3a2tosWLBg1I0E/u/GAq2trSc974477sitt96anTt3prm5+cynBQAAOA0VXdFJko6OjixfvjzNzc1ZuHBhNm7cmGPHjqW9vT1JsmzZssydOzfd3d1Jkttvvz1r167Nfffdl6amppHv8rzmNa/Ja17zmpfxTwEAAPhfFYfOddddl8OHD2ft2rXp7e3N/Pnzs3PnzpEbFDzzzDOZPPnFC0V33313jh8/nt/7vd8b9ThdXV35i7/4i7ObHgD+n6bOHdUe4RVxYP2Sao8AMCFUHDpJsmrVqqxatWrM3+3atWvUzwcOHDiTpwAAADhjr+hd1wAAAF4JQgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOIIHQAAoDhCBwAAKI7QAQAAiiN0AACA4ggdAACgOEIHAAAojtABAACKI3QAAIDiCB0AAKA4QgcAACiO0AEAAIojdAAAgOJMqfYAAD+vqXNHtUc45w6sX1LtEQCgeK7oAAAAxRE6AABAcYQOAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxRE6AABAcYQOAABQnDMKnc2bN6epqSn19fVpaWnJnj17Trn/85//fC699NLU19fniiuuyIMPPnhGwwIAAJyOikNn+/bt6ejoSFdXV/bt25d58+Zl8eLFOXTo0Jj7H3nkkVx//fX50Ic+lO9973u59tprc+211+axxx476+EBAADGMqXSEzZs2JAVK1akvb09SbJly5bs2LEj27ZtS2dn5wn7/+Zv/ia//du/nY9+9KNJkltvvTUPPfRQNm3alC1btoz5HAMDAxkYGBj5+ejRo0mS/v7+Ssc9J4YGXqj2COfc2bzWXp9T8/qcmtfn1Lw+J/dqeG0Sr89L8fqcmtfn1Lw+Jzde/js8eXGW4eHhU28crsDAwMBwTU3N8AMPPDBqfdmyZcPvete7xjynsbFx+K//+q9Hra1du3b4yiuvPOnzdHV1DSdxOBwOh8PhcDgcjjGPgwcPnrJdKrqic+TIkQwODqahoWHUekNDQ5544okxz+nt7R1zf29v70mfZ/Xq1eno6Bj5eWhoKM8991wuuuiiTJo0qZKRi9Df35/GxsYcPHgw06dPr/Y4TCDeO5wN7x/OhvcPZ8P7h1MZHh7O888/nzlz5pxyX8UfXXsl1NXVpa6ubtTaBRdcUJ1hxpHp06f7h50z4r3D2fD+4Wx4/3A2vH84mRkzZrzknopuRjBz5szU1NSkr69v1HpfX19mz5495jmzZ8+uaD8AAMDZqih0amtrs2DBgvT09IysDQ0NpaenJ62trWOe09raOmp/kjz00EMn3Q8AAHC2Kv7oWkdHR5YvX57m5uYsXLgwGzduzLFjx0buwrZs2bLMnTs33d3dSZIbb7wxv/7rv5677rorS5Ysyf3335/vfve7ueeee17ev6RgdXV16erqOuHjfPBSvHc4G94/nA3vH86G9w8vh0nDwy91X7YTbdq0KXfeeWd6e3szf/78fOpTn0pLS0uS5B3veEeampry2c9+dmT/5z//+dxyyy05cOBA3vjGN+aOO+7IO9/5zpftjwAAAPh5ZxQ6AAAA41lF39EBAACYCIQOAABQHKEDAAAUR+gAAADFETrj3ObNm9PU1JT6+vq0tLRkz5491R6JCaC7uztve9vbMm3atMyaNSvXXnttnnzyyWqPxQS1fv36TJo0KTfddFO1R2GCePbZZ/P+978/F110UaZOnZorrrgi3/3ud6s9FuPc4OBg1qxZk0suuSRTp07NG97whtx6661x3yzOlNAZx7Zv356Ojo50dXVl3759mTdvXhYvXpxDhw5VezTGuW9+85tZuXJlvvOd7+Shhx7Kz372s1xzzTU5duxYtUdjgnn00Ufzd3/3d7nyyiurPQoTxH/9139l0aJFOe+88/KVr3wl//7v/5677rorF154YbVHY5y7/fbbc/fdd2fTpk15/PHHc/vtt+eOO+7I3/7t31Z7NCYot5cex1paWvK2t70tmzZtSpIMDQ2lsbExf/qnf5rOzs4qT8dEcvjw4cyaNSvf/OY382u/9mvVHocJ4ic/+Une+ta35tOf/nQ++clPZv78+dm4cWO1x2Kc6+zszL/+67/m29/+drVHYYL53d/93TQ0NOTee+8dWXv3u9+dqVOn5h//8R+rOBkTlSs649Tx48ezd+/etLW1jaxNnjw5bW1t2b17dxUnYyI6evRokuS1r31tlSdhIlm5cmWWLFky6t9D8FK+9KUvpbm5Ob//+7+fWbNm5aqrrsrWrVurPRYTwNVXX52enp489dRTSZJ/+7d/y8MPP5zf+Z3fqfJkTFRTqj0AYzty5EgGBwfT0NAwar2hoSFPPPFElaZiIhoaGspNN92URYsW5fLLL6/2OEwQ999/f/bt25dHH3202qMwwTz99NO5++6709HRkY9//ON59NFH82d/9mepra3N8uXLqz0e41hnZ2f6+/tz6aWXpqamJoODg7nttttyww03VHs0JiihA4VbuXJlHnvssTz88MPVHoUJ4uDBg7nxxhvz0EMPpb6+vtrjMMEMDQ2lubk569atS5JcddVVeeyxx7Jlyxahwyn90z/9Uz73uc/lvvvuy1ve8pbs378/N910U+bMmeO9wxkROuPUzJkzU1NTk76+vlHrfX19mT17dpWmYqJZtWpVvvzlL+db3/pWXv/611d7HCaIvXv35tChQ3nrW986sjY4OJhvfetb2bRpUwYGBlJTU1PFCRnPLr744lx22WWj1t785jfnn//5n6s0ERPFRz/60XR2dua9731vkuSKK67ID3/4w3R3dwsdzojv6IxTtbW1WbBgQXp6ekbWhoaG0tPTk9bW1ipOxkQwPDycVatW5YEHHsjXv/71XHLJJdUeiQnkN3/zN/P9738/+/fvHzmam5tzww03ZP/+/SKHU1q0aNEJt7N/6qmn8ku/9EtVmoiJ4oUXXsjkyaP/07SmpiZDQ0NVmoiJzhWdcayjoyPLly9Pc3NzFi5cmI0bN+bYsWNpb2+v9miMcytXrsx9992XL37xi5k2bVp6e3uTJDNmzMjUqVOrPB3j3bRp0074Ptcv/MIv5KKLLvI9L17SzTffnKuvvjrr1q3Le97znuzZsyf33HNP7rnnnmqPxji3dOnS3HbbbfnFX/zFvOUtb8n3vve9bNiwIR/84AerPRoTlNtLj3ObNm3KnXfemd7e3syfPz+f+tSn0tLSUu2xGOcmTZo05vpnPvOZ/MEf/MErOwxFeMc73uH20py2L3/5y1m9enX+4z/+I5dcckk6OjqyYsWKao/FOPf8889nzZo1eeCBB3Lo0KHMmTMn119/fdauXZva2tpqj8cEJHQAAIDi+I4OAABQHKEDAAAUR+gAAADFEToAAEBxhA4AAFAcoQMAABRH6AAAAMUROgAAQHGEDgAAUByhAwAAFEfoAAAAxfkfYsDK3m2FYhIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画出每个老虎机的占比\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.bar(np.arange(bandit.n_bandits),bandit.probs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz8AAAGsCAYAAADzOBmHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAfCklEQVR4nO3de5DV9X3/8dcCsqJhly66u+yISmxTvOAlYtaN1jGVAZHYOqHtaEmCqaPTzGKL2xoh4y03UZMmjopSO6k2U6kmbdWKjS3BBmKDNyz1TtSaSoq7GC27QsYV2f398RvPdBNEFxYPy+fxmPnO7Pl+v+fs+5w5w/Cc72Vr+vv7+wMAALCXG1HtAQAAAD4I4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAijCq2gPsjL6+vmzYsCFjx45NTU1NtccBAACqpL+/P2+88UZaWloyYsSOj+0My/jZsGFDJk6cWO0xAACAPcT69etz0EEH7XCfYRk/Y8eOTfL/32BdXV2VpwEAAKqlp6cnEydOrDTCjgzL+HnnVLe6ujrxAwAAvK/LYdzwAAAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCKOqPQAAUB2HLriv2iPsdj+9ela1RwD2II78AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEQYVP4sWLcoJJ5yQsWPHprGxMWeddVbWrVs3YJ9TTz01NTU1A5Y//uM/HrDPyy+/nFmzZmW//fZLY2NjLr744rz99tu7/m4AAADexajB7Lxy5cq0t7fnhBNOyNtvv50vfvGLmT59ep555pnsv//+lf3OP//8fPnLX6483m+//So/b9u2LbNmzUpzc3N+/OMf55VXXslnP/vZ7LPPPrnqqquG4C0BAAD8qkHFz/333z/g8W233ZbGxsasWbMmp5xySmX9fvvtl+bm5u2+xr/+67/mmWeeyQ9+8IM0NTXl2GOPzVe+8pVccsklufLKKzN69OhfeU5vb296e3srj3t6egYzNgAAwK5d89Pd3Z0kaWhoGLD+9ttvzwEHHJCjjjoqCxcuzC9+8YvKttWrV2fKlClpamqqrJsxY0Z6enry9NNPb/f3LFq0KPX19ZVl4sSJuzI2AABQoEEd+fm/+vr6Mn/+/Jx00kk56qijKuv/8A//MIccckhaWlryxBNP5JJLLsm6devyj//4j0mSzs7OAeGTpPK4s7Nzu79r4cKF6ejoqDzu6ekRQAAAwKDsdPy0t7fnqaeeyoMPPjhg/QUXXFD5ecqUKZkwYUJOO+20vPjiiznssMN26nfV1tamtrZ2Z0cFAADYudPe5s2bl2XLluXf/u3fctBBB+1w39bW1iTJCy+8kCRpbm5OV1fXgH3eefxu1wkBAADsqkHFT39/f+bNm5e77rorDzzwQCZNmvSez1m7dm2SZMKECUmStra2PPnkk9m4cWNln+XLl6euri5HHHHEYMYBAAB43wZ12lt7e3uWLl2ae+65J2PHjq1co1NfX58xY8bkxRdfzNKlS3PGGWdk/PjxeeKJJ3LRRRfllFNOydFHH50kmT59eo444oh85jOfybXXXpvOzs5ceumlaW9vd2obAACw2wzqyM/NN9+c7u7unHrqqZkwYUJlufPOO5Mko0ePzg9+8INMnz49kydPzp/92Z9l9uzZuffeeyuvMXLkyCxbtiwjR45MW1tbPv3pT+ezn/3sgL8LBAAAMNQGdeSnv79/h9snTpyYlStXvufrHHLIIfnnf/7nwfxqAACAXbJLf+cHAABguBA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFGFT8LFq0KCeccELGjh2bxsbGnHXWWVm3bt2Afd588820t7dn/Pjx+dCHPpTZs2enq6trwD4vv/xyZs2alf322y+NjY25+OKL8/bbb+/6uwEAAHgXg4qflStXpr29PQ899FCWL1+erVu3Zvr06dmyZUtln4suuij33ntvvve972XlypXZsGFDPvWpT1W2b9u2LbNmzcpbb72VH//4x/mbv/mb3Hbbbbn88suH7l0BAAD8kpr+/v7+nX3yq6++msbGxqxcuTKnnHJKuru7c+CBB2bp0qX5vd/7vSTJc889l8MPPzyrV6/OiSeemO9///v55Cc/mQ0bNqSpqSlJsmTJklxyySV59dVXM3r06Pf8vT09Pamvr093d3fq6up2dnwAKNqhC+6r9gi73U+vnlXtEYDdbDBtsEvX/HR3dydJGhoakiRr1qzJ1q1bM23atMo+kydPzsEHH5zVq1cnSVavXp0pU6ZUwidJZsyYkZ6enjz99NPb/T29vb3p6ekZsAAAAAzGTsdPX19f5s+fn5NOOilHHXVUkqSzszOjR4/OuHHjBuzb1NSUzs7Oyj7/N3ze2f7Otu1ZtGhR6uvrK8vEiRN3dmwAAKBQOx0/7e3teeqpp3LHHXcM5TzbtXDhwnR3d1eW9evX7/bfCQAA7F1G7cyT5s2bl2XLlmXVqlU56KCDKuubm5vz1ltvZdOmTQOO/nR1daW5ubmyzyOPPDLg9d65G9w7+/yy2tra1NbW7syoAAAASQZ55Ke/vz/z5s3LXXfdlQceeCCTJk0asP3444/PPvvskxUrVlTWrVu3Li+//HLa2tqSJG1tbXnyySezcePGyj7Lly9PXV1djjjiiF15LwAAAO9qUEd+2tvbs3Tp0txzzz0ZO3Zs5Rqd+vr6jBkzJvX19TnvvPPS0dGRhoaG1NXV5cILL0xbW1tOPPHEJMn06dNzxBFH5DOf+UyuvfbadHZ25tJLL017e7ujOwAAwG4zqPi5+eabkySnnnrqgPW33nprzj333CTJt771rYwYMSKzZ89Ob29vZsyYkZtuuqmy78iRI7Ns2bJ8/vOfT1tbW/bff//MnTs3X/7yl3ftnQAAAOzALv2dn2rxd34AYNf5Oz/A3uAD+zs/AAAAw4X4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKMOj4WbVqVc4888y0tLSkpqYmd99994Dt5557bmpqagYsp59++oB9Xn/99cyZMyd1dXUZN25czjvvvGzevHmX3ggAAMCODDp+tmzZkmOOOSaLFy9+131OP/30vPLKK5Xl7/7u7wZsnzNnTp5++uksX748y5Yty6pVq3LBBRcMfnoAAID3adRgnzBz5szMnDlzh/vU1tamubl5u9ueffbZ3H///Xn00UczderUJMkNN9yQM844I9/4xjfS0tIy2JEAAADe02655ueHP/xhGhsb85u/+Zv5/Oc/n9dee62ybfXq1Rk3blwlfJJk2rRpGTFiRB5++OHtvl5vb296enoGLAAAAIMx5PFz+umn5zvf+U5WrFiRa665JitXrszMmTOzbdu2JElnZ2caGxsHPGfUqFFpaGhIZ2fndl9z0aJFqa+vrywTJ04c6rEBAIC93KBPe3svZ599duXnKVOm5Oijj85hhx2WH/7whznttNN26jUXLlyYjo6OyuOenh4BBAAADMpuv9X1hz/84RxwwAF54YUXkiTNzc3ZuHHjgH3efvvtvP766+96nVBtbW3q6uoGLAAAAIOx2+PnZz/7WV577bVMmDAhSdLW1pZNmzZlzZo1lX0eeOCB9PX1pbW1dXePAwAAFGrQp71t3ry5chQnSV566aWsXbs2DQ0NaWhoyJe+9KXMnj07zc3NefHFF/OFL3whv/7rv54ZM2YkSQ4//PCcfvrpOf/887NkyZJs3bo18+bNy9lnn+1ObwAAwG4z6CM/jz32WI477rgcd9xxSZKOjo4cd9xxufzyyzNy5Mg88cQT+Z3f+Z185CMfyXnnnZfjjz8+P/rRj1JbW1t5jdtvvz2TJ0/OaaedljPOOCMnn3xybrnllqF7VwAAAL9k0Ed+Tj311PT397/r9n/5l395z9doaGjI0qVLB/urAQAAdtpuv+YHAABgTyB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKMKraAwAAMLwcuuC+ao/wgfjp1bOqPQJDzJEfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIgw6flatWpUzzzwzLS0tqampyd133z1ge39/fy6//PJMmDAhY8aMybRp0/L8888P2Of111/PnDlzUldXl3HjxuW8887L5s2bd+mNAAAA7Mig42fLli055phjsnjx4u1uv/baa3P99ddnyZIlefjhh7P//vtnxowZefPNNyv7zJkzJ08//XSWL1+eZcuWZdWqVbngggt2/l0AAAC8h1GDfcLMmTMzc+bM7W7r7+/Pddddl0svvTS/+7u/myT5zne+k6amptx99905++yz8+yzz+b+++/Po48+mqlTpyZJbrjhhpxxxhn5xje+kZaWll14OwAAANs3pNf8vPTSS+ns7My0adMq6+rr69Pa2prVq1cnSVavXp1x48ZVwidJpk2blhEjRuThhx/e7uv29vamp6dnwAIAADAYQxo/nZ2dSZKmpqYB65uamirbOjs709jYOGD7qFGj0tDQUNnnly1atCj19fWVZeLEiUM5NgAAUIBhcbe3hQsXpru7u7KsX7++2iMBAADDzJDGT3Nzc5Kkq6trwPqurq7Ktubm5mzcuHHA9rfffjuvv/56ZZ9fVltbm7q6ugELAADAYAxp/EyaNCnNzc1ZsWJFZV1PT08efvjhtLW1JUna2tqyadOmrFmzprLPAw88kL6+vrS2tg7lOAAAABWDvtvb5s2b88ILL1Qev/TSS1m7dm0aGhpy8MEHZ/78+fnqV7+a3/iN38ikSZNy2WWXpaWlJWeddVaS5PDDD8/pp5+e888/P0uWLMnWrVszb968nH322e70BgAA7DaDjp/HHnssn/jEJyqPOzo6kiRz587Nbbfdli984QvZsmVLLrjggmzatCknn3xy7r///uy7776V59x+++2ZN29eTjvttIwYMSKzZ8/O9ddfPwRvBwAAYPtq+vv7+6s9xGD19PSkvr4+3d3drv8BgJ106IL7qj3CbvfTq2dVe4S9UgnfncT3Z7gYTBsMi7u9AQAA7CrxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUQfwAAABFED8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAUYcjj58orr0xNTc2AZfLkyZXtb775Ztrb2zN+/Ph86EMfyuzZs9PV1TXUYwAAAAywW478HHnkkXnllVcqy4MPPljZdtFFF+Xee+/N9773vaxcuTIbNmzIpz71qd0xBgAAQMWo3fKio0alubn5V9Z3d3fn29/+dpYuXZrf/u3fTpLceuutOfzww/PQQw/lxBNP3O7r9fb2pre3t/K4p6dnd4wNAADsxXbLkZ/nn38+LS0t+fCHP5w5c+bk5ZdfTpKsWbMmW7duzbRp0yr7Tp48OQcffHBWr179rq+3aNGi1NfXV5aJEyfujrEBAIC92JDHT2tra2677bbcf//9ufnmm/PSSy/lt37rt/LGG2+ks7Mzo0ePzrhx4wY8p6mpKZ2dne/6mgsXLkx3d3dlWb9+/VCPDQAA7OWG/LS3mTNnVn4++uij09ramkMOOSTf/e53M2bMmJ16zdra2tTW1g7ViAAAQIF2+62ux40bl4985CN54YUX0tzcnLfeeiubNm0asE9XV9d2rxECAAAYKrs9fjZv3pwXX3wxEyZMyPHHH5999tknK1asqGxft25dXn755bS1te3uUQAAgIIN+Wlvf/7nf54zzzwzhxxySDZs2JArrrgiI0eOzDnnnJP6+vqcd9556ejoSENDQ+rq6nLhhRemra3tXe/0BgAAMBSGPH5+9rOf5Zxzzslrr72WAw88MCeffHIeeuihHHjggUmSb33rWxkxYkRmz56d3t7ezJgxIzfddNNQjwEAADDAkMfPHXfcscPt++67bxYvXpzFixcP9a8GAAB4V7v9mh8AAIA9gfgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAiiB+AACAIogfAACgCOIHAAAogvgBAACKIH4AAIAiiB8AAKAI4gcAACiC+AEAAIogfgAAgCKIHwAAoAjiBwAAKIL4AQAAijCq2gMAAMDe5NAF91V7hA/ET6+eVe0RBs2RHwAAoAjiBwAAKILT3oaAQ5sAALDnEz9QZeIZAOCD4bQ3AACgCOIHAAAogvgBAACK4JofYI/mmigAYKiIHwD2WiXEs3AGeP+c9gYAABRB/AAAAEVw2hsAwHY4bRL2PuIHYBjznzMAeP+c9gYAABRB/AAAAEVw2hu7XQmn5SROzQEA2NM58gMAABRB/AAAAEUQPwAAQBGqGj+LFy/OoYcemn333Tetra155JFHqjkOAACwF6ta/Nx5553p6OjIFVdckccffzzHHHNMZsyYkY0bN1ZrJAAAYC9Wtbu9ffOb38z555+fz33uc0mSJUuW5L777stf//VfZ8GCBQP27e3tTW9vb+Vxd3d3kqSnp+eDG3gH+np/Ue0RPhA7+3n7fHbM57NjPp8dK+Hz2ZV/630+O+bz2TGfz7sr4bNJfD7vZU/5v/g7c/T397/nvjX972evIfbWW29lv/32y9///d/nrLPOqqyfO3duNm3alHvuuWfA/ldeeWW+9KUvfcBTAgAAw8X69etz0EEH7XCfqhz5+fnPf55t27alqalpwPqmpqY899xzv7L/woUL09HRUXnc19eX119/PePHj09NTc1un3dP09PTk4kTJ2b9+vWpq6ur9jgMM74/7ArfH3aW7w67wveHHenv788bb7yRlpaW99x3WPyR09ra2tTW1g5YN27cuOoMswepq6vzDwA7zfeHXeH7w87y3WFX+P7wburr69/XflW54cEBBxyQkSNHpqura8D6rq6uNDc3V2MkAABgL1eV+Bk9enSOP/74rFixorKur68vK1asSFtbWzVGAgAA9nJVO+2to6Mjc+fOzdSpU/Oxj30s1113XbZs2VK5+xvvrra2NldcccWvnAoI74fvD7vC94ed5bvDrvD9YahU5W5v77jxxhvz9a9/PZ2dnTn22GNz/fXXp7W1tVrjAAAAe7Gqxg8AAMAHpSrX/AAAAHzQxA8AAFAE8QMAABRB/AAAAEUQP8PQ4sWLc+ihh2bfffdNa2trHnnkkWqPxB5u0aJFOeGEEzJ27Ng0NjbmrLPOyrp166o9FsPU1VdfnZqamsyfP7/aozBM/M///E8+/elPZ/z48RkzZkymTJmSxx57rNpjMQxs27Ytl112WSZNmpQxY8bksMMOy1e+8pW4Xxc7S/wMM3feeWc6OjpyxRVX5PHHH88xxxyTGTNmZOPGjdUejT3YypUr097enoceeijLly/P1q1bM3369GzZsqXaozHMPProo/nLv/zLHH300dUehWHif//3f3PSSSdln332yfe///0888wz+Yu/+Iv82q/9WrVHYxi45pprcvPNN+fGG2/Ms88+m2uuuSbXXnttbrjhhmqPxjDlVtfDTGtra0444YTceOONSZK+vr5MnDgxF154YRYsWFDl6RguXn311TQ2NmblypU55ZRTqj0Ow8TmzZvz0Y9+NDfddFO++tWv5thjj811111X7bHYwy1YsCD//u//nh/96EfVHoVh6JOf/GSampry7W9/u7Ju9uzZGTNmTP72b/+2ipMxXDnyM4y89dZbWbNmTaZNm1ZZN2LEiEybNi2rV6+u4mQMN93d3UmShoaGKk/CcNLe3p5Zs2YN+DcI3ss//dM/ZerUqfn93//9NDY25rjjjstf/dVfVXsshomPf/zjWbFiRX7yk58kSf7zP/8zDz74YGbOnFnlyRiuRlV7AN6/n//859m2bVuampoGrG9qaspzzz1XpakYbvr6+jJ//vycdNJJOeqoo6o9DsPEHXfckccffzyPPvpotUdhmPmv//qv3Hzzzeno6MgXv/jFPProo/mTP/mTjB49OnPnzq32eOzhFixYkJ6enkyePDkjR47Mtm3b8rWvfS1z5syp9mgMU+IHCtPe3p6nnnoqDz74YLVHYZhYv359/vRP/zTLly/PvvvuW+1xGGb6+voyderUXHXVVUmS4447Lk899VSWLFkifnhP3/3ud3P77bdn6dKlOfLII7N27drMnz8/LS0tvj/sFPEzjBxwwAEZOXJkurq6Bqzv6upKc3NzlaZiOJk3b16WLVuWVatW5aCDDqr2OAwTa9asycaNG/PRj360sm7btm1ZtWpVbrzxxvT29mbkyJFVnJA92YQJE3LEEUcMWHf44YfnH/7hH6o0EcPJxRdfnAULFuTss89OkkyZMiX//d//nUWLFokfdoprfoaR0aNH5/jjj8+KFSsq6/r6+rJixYq0tbVVcTL2dP39/Zk3b17uuuuuPPDAA5k0aVK1R2IYOe200/Lkk09m7dq1lWXq1KmZM2dO1q5dK3zYoZNOOulXbq3/k5/8JIccckiVJmI4+cUvfpERIwb+d3XkyJHp6+ur0kQMd478DDMdHR2ZO3dupk6dmo997GO57rrrsmXLlnzuc5+r9mjswdrb27N06dLcc889GTt2bDo7O5Mk9fX1GTNmTJWnY083duzYX7k+bP/998/48eNdN8Z7uuiii/Lxj388V111Vf7gD/4gjzzySG655Zbccsst1R6NYeDMM8/M1772tRx88ME58sgj8x//8R/55je/mT/6oz+q9mgMU251PQzdeOON+frXv57Ozs4ce+yxuf7669Pa2lrtsdiD1dTUbHf9rbfemnPPPfeDHYa9wqmnnupW17xvy5Yty8KFC/P8889n0qRJ6ejoyPnnn1/tsRgG3njjjVx22WW56667snHjxrS0tOScc87J5ZdfntGjR1d7PIYh8QMAABTBNT8AAEARxA8AAFAE8QMAABRB/AAAAEUQPwAAQBHEDwAAUATxAwAAFEH8AAAARRA/AABAEcQPAABQBPEDAAAU4f8BENEyyFm8DJMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 每个老虎机的次数\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.bar(np.arange(bandit.n_bandits),bandit.bandits[:,0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用softmax策略进行选择\n",
    "def softmax(x,tau=1):\n",
    "    return np.exp(x/tau)/np.sum(np.exp(x/tau))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAGsCAYAAADAPO4TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwAElEQVR4nO3de3zU1Z3/8ffMZGZynVxJICTcUQQFLSCiWy8raK3Wbdf1Z5XdH6V9uFVhq2vrrtpthe12sdtd1q7t6na71f62D4q9YfvoqmuKgkVBAUFBFES5CSQhhGRynczl/P6YzEyGADJhkskcXs/Hg8ck3/nmO2c+GX3nnO/5nq/DGGMEAACs4cx0AwAAQHoR7gAAWIZwBwDAMoQ7AACWIdwBALAM4Q4AgGUIdwAALJMz1C8YiUR0+PBhFRUVyeFwDPXLAwCQtYwxamtrU3V1tZzOU/fPhzzcDx8+rNra2qF+WQAArHHw4EHV1NSc8vkhD/eioiJJ0Yb5fL60HDMYDOrFF1/UddddJ7fbnZZjnsuoZ/pR0/SinulHTdNrsOrp9/tVW1sbz9JTGfJwjw3F+3y+tIZ7fn6+fD4fH8o0oJ7pR03Ti3qmHzVNr8Gu58ed1mZCHQAAliHcAQCwDOEOAIBlCHcAACxDuAMAYBnCHQAAyxDuAABYhnAHAMAyhDsAAJYh3AEAsAzhDgCAZQh3AAAsQ7gDAGAZ68K9tSuoQy1dmW4GAAAZY124X/L3L+qKR19So787000BACAjrAv3iIk+bj/UmtmGAACQIVaFeziW7JI8OVa9NQAAzphVCdgVDMe/drusemsAAJwxqxKwqycR7i6nI4MtAQAgc6wK984+PfdQ2JxmTwAA7GVXuAf6hHskMmSv+/KuRr1X749/3+Dv1p7Gdu1pbFN7ICRJausO6khrl3bVt+mKR1/S42vej+/fHQwrFE5urzFGkT5zCJ7bfkT/9MJ7aunsSdovEjH633fq9dqepn7tCoUjqtvZoCOtZ3dpYIO/O6ktH6els0fG9N8/EjFqag+oJzSw303smMaYkx4fABCVk+kGpFPfc+6hjwmjrp6wjnUEVFOaL2OM3jns16TKQuW6XSm95p7GNi16apPyPS6tvPMybdl/XP/8v7vibRldkqdf3DVXX/rJZu1uaItP+vuXut3y5bkVihg9/tL7Gl2Sp7uvnih/V0jtgaBWvXFQnhyn7rpqonY1tOmJtR9Ikv597QdyOCRjJG+OU4HeoHQ6pBf/+iq9eeC4fvjKh/rcJaP1222HtauhTZVFXq34PxertSsol1P60R/2alxFga48b4TGluXr39fu0Xv1bfrUtJFa/MeTtO69BoUjUjAc0Yo17+mJtR/Ik+PU5MpClRV49Jnp1frd9iM61h6QwyEVeHLkyXHK3x3SvqYOtXYFNb6iQBNHFGrO+DK5XQ41d/Ro9bZDOtjcJW+OU9OqfbplZo1GFHr16zcPaXptsfY0tivX7VJLZ49au4IqK/BqRk2xpo7y6ffvNupnbxxQvselYx09GlWcqxsuHCVfXo7eOexXKBzRiCKvxlcUasehVvWEI6pv7db7jW06v6pI10yp1P5jnZpQUaDjnUEdaO7Q7oZ2RYxRdUme/F1BjSsvUChitP9Yh2aNK9XnZ49RTWme7ntmm4LhiCqLcpMmbZYVeNQdDKsrGNbxzh4d7whq1rhSjSsv0PiKAjkc0qZ9zfJ3hRQKh7XzA6ee3LtB4yoK5M1xqjjPrbAx6gyEtfOIX8FwRA6HQ7WleaopzZc3x6lQxMjtcuhYR496QhE1+gMaV5GvpvYe5TgdGluer/ZASB8e7dCEEQWaVl2s1q6gXt/bLJdD6giENbXap09OrlBzR486AiEd8XerNN+jzp6wLh1XpsMtXfr9uw0q8OaosyekxraAJMntdGpseb46g2G5HA5NrymWw+FQIBTWR8ejv0e3y6m27pDKCzxq6epROCIVeFxyOh0KhY0a27p1flWRjvi71dzeo/JCj7w5Lu0/1qHqkjx1BEKq93eryperGTXFChuptbNHzZ09csght8upI61dKsn3qCTfrUjEyN8dUlNbt0ItTm3/390aW1GohtZutQdC8uW51R0My+Nyqj0QUlN7QJMri9QZDOlYe49m1BSrKxhWgz+g9u6QjrYHVFbg0cW1JTrS2iVfrltul1ONbQG1dPaowJujuRPK1dQe0K6GNuV7XCr0uvV+Q5tGl+bJ43Jq37FOGWM0rqJALqdDpfkedQXDKslzqzA3R9s/apXb5VRXMKzOnpA6AmH58nLkdDjkcjp0sLlTxkgjirwaUeTVyOJc1bd2a9+xDtWU5CkYMcpzu3S0LaCuYFhTRhbJk+PUoZYuOR0O+XLdCoTCmjiiUOPKC7R5f7P2NLarotCrjt7OxZmIRCL6cJ9Tbz2/S05not9XXuhVKBzRvmOdKvS6VJLvkb87qLJ8j8oKPWr0B9QTjqipLaDSgujvyRipJxRRIBSRyyn5u6LtqCj0qrzQo6LcaPS0B6L/z6jy5UbbYIz2H+uUwyGV5XvkdkX/G4gYo0Z/QLlup7qDEVX6vHI6oqdem9oDKvTmqKUzqOJ8t4p7PwPFeW41tffIyKg4z61gyCjH5VBXT1gtXT0qznPr/JE+eVwONbYFVFnk1aTKQr11sFVHWrvU0ROW2+nQ6NI8dfaEtf9Yp3x5buU4HTrWHlClL1ddPWGFjdHkykIV57n15oHjCoaNakrzlOty6MPjDs1o6dK4Ee6UciUdHCaFLlA4HNbSpUv105/+VPX19aqurtYXvvAF/d3f/Z0cjjM7x+33+1VcXKzW1lb5fL4BN7yvYDCo5557Tr7z5mjRT7ZIkv7z/87S/KlVp/yZBT/aqFf3HNP/3nel3j3i133PbNMfT6nUj78wO6XX/vnmg/qbX7592n36hvBwd9mEMm38sFmTfBF1O/P1UQvrBQDAQN35R+P09Zumpe14Z5qhKfXcv/Od7+iJJ57QT37yE02bNk2bN2/WokWLVFxcrK985Stn3eiz1XdC3YnD3H01d/To1T3HJEm/f7dBdTsbJEkvvdcoSfrllo/08nuNmje1Up+7pOa0r7njJNfT3zarVnkel/Y2dWjd7qPxYM/3uHT5xHJNrS7W3qaOeHt//25D/GcvGl2sUcW5OtDcqffq25KO++qDf6yfbtyvprZoj/ma8yt18ZgS1bd269YnN/QbrbhjzhgtmDNG33h2h9480CIpeolgSZ473juTpBm1JTrY3Knmjh5t/LBZkrTH75TUrYpCr5rao/tOrizUrHFl8R70P986Q3lul3Ye8as9ENJ5VYXavO+4Lp9YIW+OU2991KIPmzokSTlOh66cPELzplbp6Vf36Ud/+FBtvb2KS8eX6XhHj95vbJckTRlZpC/+0Xg9u/WQNu87rp7eXvniqydqbHmBzh9ZpJ2H/fqf7Ue041CrrphUoamjfNrV0KajbQFNqiyU2+VUS2ePPjl5hP6w56iOtHSryufV0baA8r05umCUT4eOd2nd7qOaO6Fc5YUeHe/o0djyfBlJr+5p0qt7jsVPq1w7pVIX1RSrrMAjKXqKYW9Th3x5bpUXeOR1u5TvcWnrgRY1tnVrd0O7jDGaUVsiX65bLodR86EPddWll6i5M6SIMfroeLTnFQxHdHFtiUaX5ikQiui1D5p0tC2gg82d2tPYLofDoZtnVOudw63K9+SouiRPIwo9+v7LexQx0pXnjdDlE8u1dlejdje067IJZZpcWaSSfLcKvDla/36TdtW3Kdft1AdHO+LvSYqOPjgkTa8p1syxpcrz5KimNE8OSXubOrTxw2OaMsqnwy1dau7oUVmBRy6HQzWleeroCSsYjqiswKP61u54r7y5o0cRYxSKGHX1hNXg79a4igKNLctXR09YR9sCGlOWr+aOgCJGmlbt076mDr2xr1nlBV4V5eaoyperQCiiXfV+HTzepXDEaEZNcbwH75LRa2/v1oRxY7X1YKvKCz2aVl2s5o6A8twutQVC2nk4eqqsNN8jX16Oth1sUTgS/QN2VHGuCr1uVfq82tfUoZ1H/Koo9CoUMWrtCmp0Sa5y3S61dYe0Zf9xVRRGe/ddwbDau0Oq9OWquaNHDkljyvIVihit231U4YhRMBzRhBEFOtTSrUAwrKmjfHI5Hcr3uORwOOIjPqGIUVt39L+btu6QfvbGARXnuVWS71ae26ULRxerqb1H7t7e5ogirzw5Tu087JeRVFsareGB5k7VlObr4PFO7W3q0KQRhRpTnq+OQEjjygvOuOMViYT1wQcfauLECXI6oyOYprcnnet2alJlobqDER3r6FE4EtG+pk553U7le1zyd4X0ibEl6glF1NwRlNMhed1OOR0Ovf5hs8oKPLpkTImOdwbV1B6Ijyg4HQ71hCNq7ujR5MpCeXOcqirOldfl1PHOoAKhsFxOp1xOqbIoV93BsHLdLjW2dStipHDYyJeXo/ZAWFW+6EhFS2dQOS6Hjrb1qKY0TzlOh474u+V1OeV0OlTgcakwN0e76tt0vDPYOyrn1f5jnfroeJcmVRZqWrVPeR6XunrCqvd3q8CTo8re40eMVJznVmtXUAUel4yktw626KPjXbryvBEqys1RU3tArZ1B7T3SpHHl+WdU/3RLqed+0003qaqqSv/1X/8V33bLLbcoLy9PP/3pT8/oGIPZcw9WX6yv/WqHJOnfbr9EN8+o7rfv0baA1rzboAd/vV2S9OANU/TK7qN67YNo2L/795/SBd98QZLkdjn0h7/5Yz2/40g8LAq9OfrCFeMVjhjd/dMterH3D4O7r56oRn9Af37ZGF0ypjT+WrO//fv4a6/56lWaOKKwX5seXr1dK18/IEn6/f1XaVJldJ9d9W26/rFX4vvte/TGU9bgUEuXlv32Hb24s0FLrpmkL1wxThWFXknR/0Cb2qPDUN2hsHy5brUHQqpv7dJL7zXq85eO0YYPjunL/70lfryqPKMvXT1FfzF3vCLG6HdvH9anLxqloly3XvugSZVFufF2DoQxRm991KpAMKzZ48rk7L26YW9Th6pLcuXNSZwe2dfUoYoirwq9Q3sWKRiOqMHfrRFF3qT2DOhYvZ/RT3/603K7Uxuii0RMvD597Tzsl9ftTPpMGWM+9n/mxhgd7wzG/1DJRgOpZyRi5HDojMPuXHM2n1H0N1j1HJSe++WXX64f/vCH2r17t8477zy99dZbWr9+vVasWHHKnwkEAgoEEr1Evz/613QwGFQwGEzl5U8pdpy27sRks0BP/+NvP9SqP33y9aRtja1dyu2z4M3exsTEuGDY6KrvvtxvSL0sP0eHW7vjwS5JN06r1Pkji5LaU5KbPF+xtthz0vc8Z2xJPNzHlCT2mVCeq+Wfm6aHVr+jh284/7T1qizI0fc/P127G9o1qbJQLqcjaf+SXKdkwspzRdvndUpjS3O1aO4YSdI1k8v0F3Nq9d+vH9SXLh+j6eZDzZ9drRxHRHJIf3rxqPh7mz2mOOl9DtS0kQWSpHA4pHDvoEtNsUcyEQWDiZqPLvZIMmn7vKSiqtDdrz0DEWv7QN9DONx/2+QReQM+ZpHHkZF6psvZ1hP9UdP0Gqx6nunxUuq5RyIRPfzww/qnf/onuVwuhcNhffvb39ZDDz10yp9ZunSpli1b1m/7ypUrlZ+f3uGKlw879Oz+aA/r9olhXVZp9GaTQ2uPOLVwcljP7nfq7ebTXyBwx8SwVn5w+l6az23UFZaCkWgPoKbA6P6LwnKdpENw74bE30/fm3vyyS3GSBsaHRpTaFRT0P/5loBU7JGGosNxtEsqy9VJ3wsAILM6Ozt1xx13fGzPPaVwX7VqlR544AF997vf1bRp07Rt2zbdd999WrFihRYuXHjSnzlZz722tlZNTU1pHZavq6vTHu9kPb52ryTp72++QLfPrtXkb7woSbpycrnCEenV3uH3VEwf7dPbh/z9ts8ZX6qffGGWnKcZ6nt222E98KsdeuKOizXvgsqUXzsTYvWcP38+w3NpQk3Ti3qmHzVNr8Gqp9/vV0VFRXqH5R944AE9+OCD+vznPy9Juuiii7R//34tX778lOHu9Xrl9Xr7bXe73Wn/AAX6LFxj5Ew6/oHmLpWmcI7xjjljNGd8mfLcLl17QZU+8/h6HWnt0iVjSvXSe43yuJz6xz+drlzv6Y956+yxumXmmJOeMx3uBuN3dK6jpulFPdOPmqZXuut5psdKKdw7OzuTrn+UJJfLpcgQLhhzOp19ZssHT5gt39YdSmlSVE1pnv7k4tHx739x11yFwkZH/F3a09iuO6+ccNLJcSeTjcEOAMheKYX7Zz7zGX3729/WmDFjNG3aNG3dulUrVqzQF7/4xcFqX0r6hvuJl4W1BUIpnbOOzTSPKeidqV2c79Yrf3PNwBsJAMAgS2n52ccff1x/9md/pnvuuUcXXHCBvva1r+nLX/6yvvWtbw1W+1Jyuuvce0IRNbX3nPgjSb523Xnxr2tK89LbOAAAhkhKPfeioiI99thjeuyxxwapOWen741jgmFz2oVsJGlSZaH29C6cIklf+qMJumxCuXY1tOnyiRWD1k4AAAaTXTeO6Um+cUz7x6yr/JvFV+jGi0bFv891OzVrXJkWzBk7aG0EAGCwWRXugVDyLV/buk8f7gXenKThd1auAgDYwKpw7wklJtEFw0b+7o9fyWf2uLLBbBIAAEPOqlu+9r1PeCgS+dieuyRde0Gl/uXWGZpanZ4FdQAAyDS7wr3PBLrgKYbl/2xmjTZ8cExfv/ECSdGh+Ftmnv7ObwAAZBOrwj3Yt+cejqg90H9Y/oYLR+qfb50xlM0CAGBI2XXOPdx3WD7Rc5/c59akVb7cIW8XAABDydpwD4YT59z7nk8fWUy4AwDsZtWwfNKEuj6z5SsKvfrR/52lrmC437KyAADYxppwNyY6iS4mFInoqD96q9mKQq/mTa3KVNMAABhS1gzLh0+4K30wbHSktVuSNIqheADAOcSacA+dEO6hSET1/mi4c54dAHAusSfcT7hHTLTn3iWJnjsA4NxizTn3E8P9jb3N8a+5/A0AcC6xpud+4jn3GE+OU7lu19A2BgCADLIm3E885x7Tc2KXHgAAy9kT7qfI8MsmcNc3AMC5xepwdzjEOvIAgHOOPeF+kmH5L185UTWl+UPfGAAAMsiicHf021ZW4M5ASwAAyCx7wr13WN7tSoR8Sb4nQ60BACBzrAv3vD6XvZUS7gCAc5A14R67zr3Am1iXh2F5AMC5yJpwj02oy/ckeu4MywMAzkX2hHvvsLwnh2F5AMC5zbpw7zsUX5zHsDwA4Nxjz41jeoflK4ty9f++eKnyPC65nP0vjwMAwHbWhHtsQp3b5dCV543IbGMAAMggi4blo710T441bwkAgAGxJgnjE+pc3N4VAHBusyfce4fl6bkDAM511iRhoufOJDoAwLnNinDvDoa14zjn3AEAkCwJ91+9eUgtPdFwn1ZdnOHWAACQWVaEe3sgLEm6fGKZrplSmeHWAACQWVaEe0x1cV6mmwAAQMZZEe7GmEw3AQCAYcOOcO99dDBRHgAAO8I9hmwHAMCScGdUHgCABDvCvfeRYXkAAGwJ93jXnXQHAMCKcI+h5w4AgCXhzil3AAASrAj3WLrTcQcAwJJwN73pzrA8AACWhDsAAEiwItxNfFierjsAAHaEe+8jw/IAAFgS7jFkOwAAloQ7y88CAJBgR7jHr4Wj7w4AgBXhznXuAAAk2BHuvei4AwBgSbhzyh0AgAQ7wp1heQAA4uwI9/jys8Q7AABWhDsAAEiwItwZlgcAIMGOcO99ZFQeAABLwh0AACRYEe6G9WcBAIizItxjmC0PAIAl4c6EOgAAEqwI9xg67gAAWBLunHEHACDBjnDvHZd3MDAPAIAl4d77yLA8AACWhDsAAEiwItyZLQ8AQIId4R77gnQHAMCOcI9hQh0AALaEO8vPAgAQZ0W4M1seAICElMP90KFD+vM//3OVl5crLy9PF110kTZv3jwYbTtjTKgDACAhJ5Wdjx8/riuuuELXXHONnn/+eY0YMULvv/++SktLB6t9KaHnDgBAiuH+ne98R7W1tXrqqafi28aPH5/2RqXKsAAtAABxKYX7b3/7W11//fW69dZbtW7dOo0ePVr33HOP7rzzzlP+TCAQUCAQiH/v9/slScFgUMFgcIDNThYORyRJkUgkbcc8l8VqSC3Th5qmF/VMP2qaXoNVzzM9nsOYM59qnpubK0m6//77deutt2rTpk2699579eSTT2rhwoUn/ZmlS5dq2bJl/bavXLlS+fn5Z/rSp/XMh0691uDUDTVhfaqWXjwAwE6dnZ2644471NraKp/Pd8r9Ugp3j8ejWbNm6bXXXotv+8pXvqJNmzZpw4YNJ/2Zk/Xca2tr1dTUdNqGpeLrz+7Qz7cc1pKrxuneeeel5ZjnsmAwqLq6Os2fP19utzvTzbECNU0v6pl+1DS9Bquefr9fFRUVHxvuKQ3Ljxo1SlOnTk3adsEFF+hXv/rVKX/G6/XK6/X22+52u9P2hh2O6KR/l8vFhzKN0vk7QhQ1TS/qmX7UNL3SXc8zPVZKl8JdccUV2rVrV9K23bt3a+zYsakcZhD03vKV6fIAAKQW7n/913+tjRs36h//8R+1Z88erVy5Uj/84Q+1ePHiwWpfSoh2AABSDPfZs2dr9erV+tnPfqYLL7xQ3/rWt/TYY49pwYIFg9W+M8LqswAAJKR0zl2SbrrpJt10002D0ZYBY/lZAAAS7FhbnuVnAQCIsyLcAQBAghXhbpgtDwBAnB3hzoQ6AADi7Aj33kc67gAAWBLuAAAgwY5wN7Fz7hluBwAAw4AV4Z64FI50BwDAjnDvfaTnDgCAJeEOAAASrAh3VqgDACDBjnBnERsAAOKsCHcAAJBgRbizQh0AAAl2hHvvI6PyAABYEu5iQh0AAHF2hDsAAIizItyZLQ8AQIId4c6wPAAAcXaEe+8jHXcAACwJdwAAkGBFuJvYLV8z3A4AAIYDO8I99gXj8gAA2BHuAAAgwYpwZ7Y8AAAJVoR7DKPyAABYEu6JCXWkOwAAVoQ7AABIsCLcWcQGAIAEO8KdCXUAAMTZEe7xG8dkuCEAAAwDVoQ7AABIsCLcTWKJukw2AwCAYcGOcO99ZFgeAABLwh0AACTYEe7MlgcAIM6KcGe2PAAACXaEe7znTroDAGBFuAMAgAQrwp3Z8gAAJNgR7kyoAwAgzo5wj0+XJ94BALAi3AEAQIId4c6wPAAAcVaEOxPqAABIsCLcAQBAghXhbnqny9NxBwDAlnDvfXQwLg8AgCXhzoQ6AADirAh3AACQYEW4M1seAIAEO8I9Ni4PAADsCHcAAJBgVbgzWx4AAEvCndnyAAAk2BHuvY903AEAsCTcAQBAghXhnlh+lq47AAB2hHvvI8PyAADYEu5MqAMAIM6KcAcAAAlWhLsRXXcAAGKsCPd4tnPSHQAAO8KdleUBAEiwItxj6LcDAGBJuMdny5PuAABYEu6KLWIDAACsCHcAAJBgRbgbZssDABBnV7hnthkAAAwLdoR77AvSHQAAO8IdAAAk2BHu3PIVAIA4K8KdW74CAJBwVuH+6KOPyuFw6L777ktTcwbGsP4sAABxAw73TZs26T/+4z80ffr0dLbnrNBxBwBAyhnID7W3t2vBggX6z//8T/3DP/zDafcNBAIKBALx7/1+vyQpGAwqGAwO5OX7ifR23cPhcNqOeS6L1ZBapg81TS/qmX7UNL0Gq55nejyHMakPai9cuFBlZWX613/9V1199dW6+OKL9dhjj51036VLl2rZsmX9tq9cuVL5+fmpvvRJLd/mUn2XQ0umhjW5mDF6AICdOjs7dccdd6i1tVU+n++U+6Xcc1+1apXefPNNbdq06Yz2f+ihh3T//ffHv/f7/aqtrdV111132oal4nvvr5e6OjVz5kz90XmVaTnmuSwYDKqurk7z58+X2+3OdHOsQE3Ti3qmHzVNr8GqZ2z0++OkFO4HDx7Uvffeq7q6OuXm5p7Rz3i9Xnm93n7b3W532t5wbNlZtzuHD2UapfN3hChqml7UM/2oaXqlu55neqyUwn3Lli1qbGzUJz7xifi2cDisV155Rd///vcVCATkcrlSa2kaMFseAICElML92muv1fbt25O2LVq0SFOmTNHf/u3fZiTYo3oXsWG6PAAAqYV7UVGRLrzwwqRtBQUFKi8v77cdAABkhh0r1MXvCkfXHQCAAV3n3tfatWvT0Iyzw/KzAAAkWNVzBwAAloR7DB13AAAsCXcTny1PvAMAYEe4xyfUAQAAO8I90w0AAGAYsSLc4+i6AwBgSbj3jsuT7QAAWBLuDMsDAJBgRbjHMFseAABLwp3Z8gAAJNgR7r2PdNwBALAl3Fl/FgCAOCvCPYa7wgEAYEm4MywPAECCFeHOtXAAACRYEe5kOwAACVaEewzD8gAAWBLuJr78LOkOAIAd4Z7pBgAAMIxYEe4xDMsDAGBJuLP8LAAACXaEe+/APD13AABsCXdOugMAEGdFuMcwWx4AAMvCnWwHAMCScGdCHQAACXaEO1e6AwAQZ0W4xziYLg8AgB3hzrA8AAAJdoR7phsAAMAwYkW4xzAqDwCAJeEeH5Yn3AEAsCTcxS1fAQCIsSLcOekOAECCHeEeQ8cdAAA7wj3WcSfbAQCwJdwNt3wFACDGjnDPdAMAABhGrAj3GGbLAwBgSbhznTsAAAl2hHumGwAAwDBiRbjH0HEHAMCScE/MlifeAQCwItwBAECCFeFuOOkOAECcFeEew6g8AACWhDvLzwIAkGBHuDOhDgCAODvCPdMNAABgGLEi3GPotwMAYEm4s/wsAAAJVoQ7AABIsCrc6bgDAGBBuJu+K9gwLg8AgA3hnviaaAcAwIZwz3QDAAAYZrI+3PtiVB4AAAvCve85dwcD8wAAWBDumW4AAADDTPaHO5PlAQBIkvXh3hfZDgCABeFu+gzM03MHAMCGcOekOwAASbI+3JPRdQcAwKpwZ1geAAALwp3lZwEASJb94c6V7gAAJMn6cO+LYXkAACwI9+RhedIdAIDsD/dMNwAAgGEm+8PdsIgNAAB9ZX2490W2AwBgQbgnDcvTdQcAILVwX758uWbPnq2ioiJVVlbqs5/9rHbt2jVYbTsjLD8LAECylMJ93bp1Wrx4sTZu3Ki6ujoFg0Fdd9116ujoGKz2fTwWsQEAIElOKju/8MILSd8//fTTqqys1JYtW3TllVemtWEDwag8AAAphvuJWltbJUllZWWn3CcQCCgQCMS/9/v9kqRgMKhgMHg2Ly9J6ulzjFAoqGAw66cRZFzs95KO3w+iqGl6Uc/0o6bpNVj1PNPjOYwZ2FnrSCSim2++WS0tLVq/fv0p91u6dKmWLVvWb/vKlSuVn58/kJdO0hGUHt4c/RtlxWUhuei9AwAs1dnZqTvuuEOtra3y+Xyn3G/A4X733Xfr+eef1/r161VTU3PK/U7Wc6+trVVTU9NpG3amjnf26NLlayVJ2//uauV6PWd9zHNdMBhUXV2d5s+fL7fbnenmWIGaphf1TD9qml6DVU+/36+KioqPDfcBDcsvWbJEv/vd7/TKK6+cNtglyev1yuv19tvudrvT8oZzchJ/m3jSdExEpet3hARqml7UM/2oaXqlu55neqyUwt0Yo7/6q7/S6tWrtXbtWo0fP35AjUunAQ48AABgrZTCffHixVq5cqV+85vfqKioSPX19ZKk4uJi5eXlDUoDP07faGe2PAAAKV7n/sQTT6i1tVVXX321Ro0aFf/3zDPPDFb7UuIg3QEASH1YfrgZhk0CACCjsv6icMNNXwEASJL14R7LdgchDwCAJBvCHQAAJMn6cI/115lKBwBAVPaHO6PxAAAkyfpwj6PrDgCAJAvCPTZbnmwHACAq+8OdYXkAAJJkf7hnugEAAAwzWR/uMQzLAwAQlfXhHlsSl3AHACDKgnDPdAsAABhesj7c4+i6AwAgyaJwJ9sBAIjK+nBnWB4AgGTZH+5cDAcAQJKsD/cYhuUBAIjK+nA33BYOAIAk2R/umW4AAADDTPaHO4vYAACQJOvDPYZwBwAgKuvDnWF5AACSZX+4k+4AACTJ+nCP9d0ZlgcAIMqCcO9FugMAIMmCcGdYHgCAZNkf7r2PdNwBAIjK+nAHAADJsj7cY8Py9NwBAIjK/nDnSncAAJJkf7hz0h0AgCRZH+4xZDsAAFFZH+6ccwcAIFn2hzvn3AEASJL94U62AwCQJOvDPYZheQAAoqwJdwAAEJX14c6lcAAAJMv6cI8h2wEAiMr6cGe2PAAAybI/3Ml2AACSZH+49z4yLA8AQFTWh3uMg3QHAECSBeFuGJcHACBJ9od7phsAAMAwk/3hzo1jAABIkvXhDgAAklkQ7gzMAwDQV9aHO8PyAAAky/5wj31BugMAIMmCcI8h2wEAiMr6cOcydwAAklkQ7tF0p+cOAEBU1oc7AABIlvXhzqg8AADJsj/cY5fCMS4PAIAkG8KdvjsAAEmyPtxj6LgDABCV/eFOxx0AgCRZH+5kOwAAybI/3FlbHgCAJFkf7nGkOwAAkiwId2bLAwCQLPvDnWF5AACSZH24xxDuAABEZX24MygPAECy7A937vkKAECS7A/33kfWlgcAICrrwx0AACTL/nBnVB4AgCRZH+6x69wZlQcAICr7w52eOwAASbI+3GPouQMAEJX14U7PHQCAZAMK9x/84AcaN26ccnNzNWfOHL3xxhvpbtcZ41I4AACSpRzuzzzzjO6//3498sgjevPNNzVjxgxdf/31amxsHIz2AQCAFKUc7itWrNCdd96pRYsWaerUqXryySeVn5+vH//4x4PRvo/FCnUAACTLSWXnnp4ebdmyRQ899FB8m9Pp1Lx587Rhw4aT/kwgEFAgEIh/7/f7JUnBYFDBYHAgbU4SCoUlRSfUpeN4SNSReqYPNU0v6pl+1DS9BqueZ3q8lMK9qalJ4XBYVVVVSdurqqr03nvvnfRnli9frmXLlvXb/uKLLyo/Pz+Vlz+p3a0Ojcp3qiLXqK6u7qyPhwTqmX7UNL2oZ/pR0/RKdz07OzvPaL+Uwn0gHnroId1///3x7/1+v2pra3XdddfJ5/Od9fE/LWlxMKi6ujrNnz9fbrf7rI95rgtSz7SjpulFPdOPmqbXYNUzNvr9cVIK94qKCrlcLjU0NCRtb2ho0MiRI0/6M16vV16vt992t9ud9g/QYBzzXEY904+aphf1TD9qml7prueZHiulCXUej0czZ87UmjVr4tsikYjWrFmjuXPnptZCAAAwKFIelr///vu1cOFCzZo1S5deeqkee+wxdXR0aNGiRYPRPgAAkKKUw/22227T0aNH9c1vflP19fW6+OKL9cILL/SbZAcAADJjQBPqlixZoiVLlqS7LQAAIA2yfm15AACQjHAHAMAyhDsAAJYh3AEAsAzhDgCAZQh3AAAsQ7gDAGAZwh0AAMsQ7gAAWGbQb/l6ImOMpDO/bd2ZCAaD6uzslN/v525GaUA904+aphf1TD9qml6DVc9Ydsay9FSGPNzb2tokSbW1tUP90gAAWKGtrU3FxcWnfN5hPi7+0ywSiejw4cMqKiqSw+FIyzH9fr9qa2t18OBB+Xy+tBzzXEY904+aphf1TD9qml6DVU9jjNra2lRdXS2n89Rn1oe85+50OlVTUzMox/b5fHwo04h6ph81TS/qmX7UNL0Go56n67HHMKEOAADLEO4AAFjGinD3er165JFH5PV6M90UK1DP9KOm6UU904+aplem6znkE+oAAMDgsqLnDgAAEgh3AAAsQ7gDAGAZwh0AAMsQ7gAAWCbrw/0HP/iBxo0bp9zcXM2ZM0dvvPFGpps0bL3yyiv6zGc+o+rqajkcDj377LNJzxtj9M1vflOjRo1SXl6e5s2bp/fffz9pn+bmZi1YsEA+n08lJSX60pe+pPb29iF8F8PH8uXLNXv2bBUVFamyslKf/exntWvXrqR9uru7tXjxYpWXl6uwsFC33HKLGhoakvY5cOCAbrzxRuXn56uyslIPPPCAQqHQUL6VYeGJJ57Q9OnT4yt6zZ07V88//3z8eWp5dh599FE5HA7dd9998W3UNDVLly6Vw+FI+jdlypT488OqniaLrVq1yng8HvPjH//YvPPOO+bOO+80JSUlpqGhIdNNG5aee+458/Wvf938+te/NpLM6tWrk55/9NFHTXFxsXn22WfNW2+9ZW6++WYzfvx409XVFd/nU5/6lJkxY4bZuHGj+cMf/mAmTZpkbr/99iF+J8PD9ddfb5566imzY8cOs23bNvPpT3/ajBkzxrS3t8f3ueuuu0xtba1Zs2aN2bx5s7nsssvM5ZdfHn8+FAqZCy+80MybN89s3brVPPfcc6aiosI89NBDmXhLGfXb3/7W/M///I/ZvXu32bVrl3n44YeN2+02O3bsMMZQy7PxxhtvmHHjxpnp06ebe++9N76dmqbmkUceMdOmTTNHjhyJ/zt69Gj8+eFUz6wO90svvdQsXrw4/n04HDbV1dVm+fLlGWxVdjgx3CORiBk5cqT57ne/G9/W0tJivF6v+dnPfmaMMWbnzp1Gktm0aVN8n+eff944HA5z6NChIWv7cNXY2GgkmXXr1hljovVzu93mF7/4RXyfd99910gyGzZsMMZE/+ByOp2mvr4+vs8TTzxhfD6fCQQCQ/sGhqHS0lLzox/9iFqehba2NjN58mRTV1dnrrrqqni4U9PUPfLII2bGjBknfW641TNrh+V7enq0ZcsWzZs3L77N6XRq3rx52rBhQwZblp327t2r+vr6pHoWFxdrzpw58Xpu2LBBJSUlmjVrVnyfefPmyel06vXXXx/yNg83ra2tkqSysjJJ0pYtWxQMBpNqOmXKFI0ZMyapphdddJGqqqri+1x//fXy+/165513hrD1w0s4HNaqVavU0dGhuXPnUsuzsHjxYt14441JtZP4fA7U+++/r+rqak2YMEELFizQgQMHJA2/eg75XeHSpampSeFwOKlIklRVVaX33nsvQ63KXvX19ZJ00nrGnquvr1dlZWXS8zk5OSorK4vvc66KRCK67777dMUVV+jCCy+UFK2Xx+NRSUlJ0r4n1vRkNY89d67Zvn275s6dq+7ubhUWFmr16tWaOnWqtm3bRi0HYNWqVXrzzTe1adOmfs/x+UzdnDlz9PTTT+v888/XkSNHtGzZMn3yk5/Ujh07hl09szbcgeFk8eLF2rFjh9avX5/ppmS1888/X9u2bVNra6t++ctfauHChVq3bl2mm5WVDh48qHvvvVd1dXXKzc3NdHOscMMNN8S/nj59uubMmaOxY8fq5z//ufLy8jLYsv6ydli+oqJCLper30zEhoYGjRw5MkOtyl6xmp2uniNHjlRjY2PS86FQSM3Nzed0zZcsWaLf/e53evnll1VTUxPfPnLkSPX09KilpSVp/xNrerKax54713g8Hk2aNEkzZ87U8uXLNWPGDH3ve9+jlgOwZcsWNTY26hOf+IRycnKUk5OjdevW6d/+7d+Uk5OjqqoqanqWSkpKdN5552nPnj3D7jOateHu8Xg0c+ZMrVmzJr4tEolozZo1mjt3bgZblp3Gjx+vkSNHJtXT7/fr9ddfj9dz7ty5amlp0ZYtW+L7vPTSS4pEIpozZ86QtznTjDFasmSJVq9erZdeeknjx49Pen7mzJlyu91JNd21a5cOHDiQVNPt27cn/dFUV1cnn8+nqVOnDs0bGcYikYgCgQC1HIBrr71W27dv17Zt2+L/Zs2apQULFsS/pqZnp729XR988IFGjRo1/D6jaZ2eN8RWrVplvF6vefrpp83OnTvNX/7lX5qSkpKkmYhIaGtrM1u3bjVbt241ksyKFSvM1q1bzf79+40x0UvhSkpKzG9+8xvz9ttvmz/5kz856aVwl1xyiXn99dfN+vXrzeTJk8/ZS+HuvvtuU1xcbNauXZt0aUxnZ2d8n7vuusuMGTPGvPTSS2bz5s1m7ty5Zu7cufHnY5fGXHfddWbbtm3mhRdeMCNGjDgnLzV68MEHzbp168zevXvN22+/bR588EHjcDjMiy++aIyhlunQd7a8MdQ0VV/96lfN2rVrzd69e82rr75q5s2bZyoqKkxjY6MxZnjVM6vD3RhjHn/8cTNmzBjj8XjMpZdeajZu3JjpJg1bL7/8spHU79/ChQuNMdHL4b7xjW+Yqqoq4/V6zbXXXmt27dqVdIxjx46Z22+/3RQWFhqfz2cWLVpk2traMvBuMu9ktZRknnrqqfg+XV1d5p577jGlpaUmPz/ffO5znzNHjhxJOs6+ffvMDTfcYPLy8kxFRYX56le/aoLB4BC/m8z74he/aMaOHWs8Ho8ZMWKEufbaa+PBbgy1TIcTw52apua2224zo0aNMh6Px4wePdrcdtttZs+ePfHnh1M9uZ87AACWydpz7gAA4OQIdwAALEO4AwBgGcIdAADLEO4AAFiGcAcAwDKEOwAAliHcAQCwDOEOAIBlCHcAACxDuAMAYJn/D+YfSOhIV93/AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class SoftmaxBandit(Bandit):\n",
    "    def __init__(self,n_bandits,eps=0.2):\n",
    "        super().__init__(n_bandits,eps)\n",
    "\n",
    "    def choose_action(self):\n",
    "        probs = softmax(self.bandits[:,1],tau=1)\n",
    "        action = np.random.choice(np.arange(self.n_bandits),p=probs)\n",
    "        return action\n",
    "\n",
    "softmax_bandit = SoftmaxBandit(10)\n",
    "\n",
    "soft_max_results = [0]\n",
    "\n",
    "for i in range(500):\n",
    "    action = softmax_bandit.choose_action()\n",
    "    reward = softmax_bandit.get_reward(softmax_bandit.probs[action])\n",
    "    softmax_bandit.step(reward,action)\n",
    "    mean_rewards = ((i+1)*rewards[-1]+reward)/(i+2)\n",
    "    soft_max_results.append(mean_rewards)\n",
    "\n",
    "plot_result(soft_max_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型代替策略\n",
    "# 设置环境，通过一个类进行实现\n",
    "class ContextualBandit():\n",
    "    def __init__(self,n_arms):\n",
    "        self.arms = n_arms\n",
    "        self.init_distribution(self.arms)\n",
    "        self.update_state()\n",
    "    \n",
    "    def init_distribution(self,arms):\n",
    "        self.bandit_matrix = np.random.rand(arms,arms) # 初始化一个随机的矩阵\n",
    "\n",
    "    def reward(self,prob): # target 是概率\n",
    "        reward = 0\n",
    "        for i in range(self.arms):\n",
    "            if np.random.rand()<prob:\n",
    "                reward +=1\n",
    "        return reward\n",
    "    \n",
    "    def get_state(self):\n",
    "        return self.state\n",
    "    \n",
    "    def update_state(self):\n",
    "        self.state = np.random.randint(0,self.arms)\n",
    "\n",
    "    def get_reward(self,arm):\n",
    "        return self.reward(self.bandit_matrix[self.get_state()][arm])\n",
    "    \n",
    "    def choose_arm(self,arm):\n",
    "        reward = self.get_reward(arm)\n",
    "        self.update_state()\n",
    "        return reward\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAGsCAYAAABEjHTMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjHUlEQVR4nO3df3BU1f3/8ddusrtJIJsEAgmBAFFAlJ8KmEb8OaRQRav9dlp0aL8UO/6EKsWi2Bb4MGqD2mFQS1HbUfr9zghqW9SvIpMMShDlt4AgiqAoVExihGQTApsle75/WFbXBCFwNzfhPB8zmXHPPTnn7DtX7mvv3t3rMcYYAQAA63jdXgAAAHAHIQAAAEsRAgAAsBQhAAAASxECAACwFCEAAABLEQIAALBUstsL+K5oNKoDBw4oPT1dHo/H7eUAANBhGGNUV1envLw8eb0nf53f7kLAgQMHlJ+f7/YyAADosPbv369evXqdtF+7CwHp6emSvn4CwWDQkTEjkYhKS0s1duxY+Xw+R8a0HTV1HjV1FvV0HjV1ViLqGQqFlJ+fHzuWnky7CwHH3wIIBoOOhoC0tDQFg0F2XIdQU+dRU2dRT+dRU2clsp6n+nY6FwYCAGApQgAAAJYiBAAAYClCAAAAliIEAABgKUIAAACWIgQAAGApQgAAAJYiBAAAYClCAAAAliIEAABgqVaHgNWrV+u6665TXl6ePB6PXnrppbjtxhjNnj1bPXr0UGpqqoqLi7V7926n1gsAABzS6hBw+PBhDRs2TAsXLmxx+yOPPKLHH39cTz75pNavX69OnTpp3LhxOnr06BkvFgAAOMdjjDGn/csej5YtW6YbbrhB0tdnAfLy8nTPPffod7/7nSSptrZWOTk5Wrx4sW688caTjhkKhZSRkaHa2lrH7iLY2NioaX9boW11nfR5zVGl+ZPU0NgkSfIne3XD8Dx98EWdIk1RfVhRpxF9srSrok7D8zNVXR/WvoMNsf45wYCORqIqLOii0p2VkqQ/jj9fT5Z/rAE56Xrn4690Wf9sbd1Xo/FDe+hQQ6OORqJ697NDqgsfkyT1zExVXmaKRvXtog17D2rL/ho1RY16d0nTvoMNyu7sV3V9oyTpZyN6aeWHVTp4uLHF55bfJVVf1oV1NBJtcfvxsXpmpurzmiNx287t1kmRJqMkr0fd0wM61NCoitqjCh09FutzYe9MbdlXI0maWNhbb+2u1r6DDfJ6pGgLe07nQLIyUn36sj6sxmPxa7puWJ7+37YD3/enUrLXo2MtDXya/MleHWuKxtaaHkiO/R2O65mZqvSUZH1YUaeC7E7aW304tu3bf4vWGpafqW37a0536Y7p5E/S4f/uv07J75Kq/xw6otP/1+MbGak+1R6JnHB79/SAQkcjJ9zHj6/nSGNU1fXhFrefbL9yer9rje7pAVXVfbPuPl3T9FV9o+q/s58e5/EoVvesNJ8ONbRcu3O6ddJX9Y3fW1t8P3+SV41NJ97vWnKy/VmSkrweNUWNfEke/arfMd37i6sdvZVwa46hjoaATz75ROeee662bNmi4cOHx/pdccUVGj58uB577LFmY4TDYYXD3/wPcPxeyNXV1Y6FgBEPvRF3YAMAoL1Y/L+Ha3T/7o6MFQqFlJ2dfcohINmRWf+roqJCkpSTkxPXnpOTE9v2XSUlJZo7d26z9tLSUqWlpTmyrtBRR58mAACOefiVd3X7+a0743AiDQ0Nrerv+tHx/vvv1/Tp02OPj58JGDt2rGNnAu5eW+rIOAAAOC07O1vXXDPSkbFCoVCr+jsaAnJzcyVJlZWV6tGjR6y9srIy7u2BbwsEAgoEAs3afT6fY++RAADQXnk9XseOd60dx9HvCSgoKFBubq5WrlwZawuFQlq/fr2KioqcnAoAgLOCkTsXpUqncSagvr5ee/bsiT3eu3evtm7dqi5duqh3796aNm2aHnzwQfXv318FBQWaNWuW8vLyYhcPAgCA9qHVIWDTpk266qqrYo+Pv58/adIkLV68WPfee68OHz6sW2+9VTU1Nbr00ku1YsUKpaSkOLdqAABwxs7oI4KJkIjvCeg78zVHxgEAwGmX9++q//PrHzgyVmuPodw7AAAAF7n5UpwQAACApQgBAABYihAAAIClCAEAALjIzavzCQEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAALjI4+LchAAAAFzE9wQAAIA2RwgAAMBShAAAACxFCAAAwEXGxYsCCAEAALjIuHhpICEAAABLEQIAALAUIQAAAEsRAgAAcBMXBgIAgLZGCAAAwE0u3jyAEAAAgKUIAQAAuIlrAgAAQFsjBAAAYClCAAAAbuLCQAAALMU1AQAAoK0RAgAAsBQhAAAASxECAABwkYuXBBACAACwFSEAAABLEQIAALAUIQAAAEsRAgAAsBQhAAAASxECAABwkTHufUiQEAAAgIv4ngAAANDmCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAALjIxe8KIgQAAGArQgAAAJYiBAAA4CKPx725CQEAALiIawIAAECbIwQAAGApQgAAAJYiBAAA4CIXLwkgBAAAYCvHQ0BTU5NmzZqlgoICpaam6txzz9UDDzwg4+bljwAAoJlkpwd8+OGHtWjRIv3jH//QoEGDtGnTJk2ePFkZGRm66667nJ4OAACcJsdDwDvvvKPrr79e48ePlyT17dtXS5Ys0YYNG5yeCgCADs/NM+WOh4BLLrlETz/9tD766CMNGDBA27Zt05o1azR//vwW+4fDYYXD4djjUCgkSYpEIopEIk4vDwCAdsUY49jxrrXjOB4CZs6cqVAopIEDByopKUlNTU166KGHNHHixBb7l5SUaO7cuc3aS0tLlZaW5tCqHH+aAAA4oiF0SMuXL3dmrIaGVvX3GIfPQyxdulQzZszQo48+qkGDBmnr1q2aNm2a5s+fr0mTJjXr39KZgPz8fFVXVysYDDqypv6zSh0ZBwAAp91xeV9N/+EAR8YKhULKzs5WbW3tKR1DHX+JPGPGDM2cOVM33nijJGnIkCH67LPPVFJS0mIICAQCCgQCzdp9Pp98Pp/TywMAoF3xer2OHe9aO47jHxFsaGiQ1xs/bFJSkqLRqNNTAQCAM+D4mYDrrrtODz30kHr37q1BgwZpy5Ytmj9/vm6++WanpwIAoMNz8U7CzoeAJ554QrNmzdKdd96pqqoq5eXl6bbbbtPs2bOdngoAAJwBx0NAenq6FixYoAULFjg9NAAAZx3uHQAAANocIQAAAEsRAgAAsBQhAAAAF7l5k11CAAAAliIEAABgKUIAAACWIgQAAOAi4+I3BRACAACwFCEAAABLEQIAALAUIQAAABd5XLyPICEAAABLEQIAALAUIQAAAEsRAgAAcBHfEwAAANocIQAAAEsRAgAAsBQhAAAAN7l3SQAhAAAAWxECAACwFCEAAABLEQIAAHCRi5cEEAIAALAVIQAAAEsRAgAAcJF7NxImBAAA4KpOgWTX5iYEAADgouuH93BtbkIAAAAu8nnde0OAEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAALjJw70DAABAGyMEAABgKUIAAACWIgQAAGApQgAAAJYiBAAAYClCAAAAliIEAABgKUIAAACWIgQAAGApQgAAAJYiBAAAYClCAAAAliIEAABgKUIAAAAucu9GwoQAAACsRQgAAMBSCQkBn3/+uX7xi1+oa9euSk1N1ZAhQ7Rp06ZETAUAAE5TstMDHjp0SKNHj9ZVV12l119/Xd26ddPu3buVlZXl9FQAAOAMOB4CHn74YeXn5+vZZ5+NtRUUFDg9DQAAOEOOh4BXXnlF48aN089+9jOVl5erZ8+euvPOO3XLLbe02D8cDiscDsceh0IhSVIkElEkEnF6eQAAtCvHjjl3vGvtOI6HgE8++USLFi3S9OnT9fvf/14bN27UXXfdJb/fr0mTJjXrX1JSorlz5zZrLy0tVVpamkOrcvxpAgDgiFWrytXZ58xYDQ0NrervMcYYZ6b+mt/v18iRI/XOO+/E2u666y5t3LhRa9eubda/pTMB+fn5qq6uVjAYdGRN/WeVOjIOAABOe/t3o9U9o5MjY4VCIWVnZ6u2tvaUjqGOv0Tu0aOHLrjggri2888/X//6179a7B8IBBQIBJq1+3w++XwORSMAANqp5GTnjnetHcfxjwiOHj1au3btimv76KOP1KdPH6enAgAAZ8DxEPDb3/5W69at05/+9Cft2bNHzz33nJ5++mlNmTLF6akAAMAZcDwEjBo1SsuWLdOSJUs0ePBgPfDAA1qwYIEmTpzo9FQAAOAMJOSy+WuvvVbXXnttIoYGAAAO4d4BAABYihAAAICLPC7eS5gQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAAC4yCP3biNICAAAwFKEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwFKEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAAFxkZFybmxAAAICLou5lAEIAAAC2IgQAAOAi924kTAgAAMBahAAAACxFCAAAwFKEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwEUufmswIQAAAFsRAgAAsBQhAAAASxECAACwFCEAAABLEQIAAHARtxIGAABtjhAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACAi7h3AAAAaHOEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwFIJDwHz5s2Tx+PRtGnTEj0VAABohYSGgI0bN+qpp57S0KFDEzkNAAAd1ll5K+H6+npNnDhRf/vb35SVlZWoaQAAwGlKTtTAU6ZM0fjx41VcXKwHH3zwhP3C4bDC4XDscSgUkiRFIhFFIpFELQ8AgHbByeNda8dJSAhYunSp3n33XW3cuPGkfUtKSjR37txm7aWlpUpLS3NoRQnLOgAAnJHy8nJ19jkzVkNDQ6v6e4wxjt67YP/+/Ro5cqTKyspi1wJceeWVGj58uBYsWNCsf0tnAvLz81VdXa1gMOjImvrPKnVkHAAAnLbmntHKyezkyFihUEjZ2dmqra09pWOo4y+RN2/erKqqKl100UWxtqamJq1evVp/+ctfFA6HlZSUFNsWCAQUCASajePz+eTzORSNAABop5IdPN61dhzHQ8CYMWO0ffv2uLbJkydr4MCBuu++++ICAAAAcI/jISA9PV2DBw+Oa+vUqZO6du3arB0AALiHbwwEAMBSbXLZ/KpVq9piGgAA0AqcCQAAwFKEAAAALEUIAADARWflvQMAAED7RggAAMBShAAAACxFCAAAwFKEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwFKEAAAALEUIAADAUoQAAAAsRQgAAMBFxsW5CQEAAFiKEAAAgKUIAQAAuMjj4tyEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwFKEAAAALEUIAADARXxtMAAAaHOEAAAALEUIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwEXcShgAALQ5QgAAAJYiBAAAYClCAAAAliIEAADgIu4dAAAA2hwhAAAASxECAACwFCEAAABLEQIAALAUIQAAAEsRAgAAsBQhAAAASxECAACwFCEAAAAXcSthAADQ5ggBAAC4iHsHAACANkcIAADAUoQAAAAsRQgAAMBShAAAACxFCAAAwFKEAAAALOV4CCgpKdGoUaOUnp6u7t2764YbbtCuXbucngYAAJwhx0NAeXm5pkyZonXr1qmsrEyRSERjx47V4cOHnZ4KAACcgWSnB1yxYkXc48WLF6t79+7avHmzLr/8cqenAwAAp8nxEPBdtbW1kqQuXbq0uD0cDiscDsceh0IhSVIkElEkEkn08gAAcFW06Zhjx7vWjpPQEBCNRjVt2jSNHj1agwcPbrFPSUmJ5s6d26y9tLRUaWlpDq0k4VkHAIBTcklOVO9UfvNu/MY1qxwbu6GhoVX9PcaYhN274I477tDrr7+uNWvWqFevXi32aelMQH5+vqqrqxUMBh1ZR/9ZpY6MAwDAmdr9wNi449LO2VfJ5/M5MnYoFFJ2drZqa2tP6RiasJfIU6dO1auvvqrVq1efMABIUiAQUCAQaNbu8/kcKwoAAO3Fd49tTh7vWjuO4yHAGKPf/OY3WrZsmVatWqWCggKnpwAAAA5wPARMmTJFzz33nF5++WWlp6eroqJCkpSRkaHU1FSnpwMAAKfJ8e8JWLRokWpra3XllVeqR48esZ/nn3/e6akAAMAZSMjbAQAAoP3j3gEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAIB2JzPN5/YSztiovlluL+GkCAEA0E70ykpt9e8MzE3XH8efH3v81r1XnfR3uqUHYv99ybldT32unM4K+kzrFniakr3fHJ4u7J15yr83Y9x5CVjNN1bec0WztiSvp1nbs78apRdvvySha3ECIQAA2glP82MJkFCEAACwGMHDboQAAAAsRQgAAMBShAAAACxFCAAAi3nU/i8KMG3zgQQrEQIAwDLt/7DfsZgOnFIIAQAAWIoQAAAW4yOCdiMEAADatfZ+sr29r+/7EAIAALAUIQAA2om2ulK/o71y5R2LxCEEAABgKUIAAKAd6mjnKzomQgAAnEVae7W/h48HtEoH/kqAFhECAMAyHPZxHCEAACzT0V78n2UvvtsVQgAAAJYiBAAAYClCAAB0YGfbhWodUUf+GxACAACwFCEAACzWwa4RhMMIAQDQTjhx1T6f+0drEAIAALAUIQAALMaJA7sRAgAAsBQhAAAASxECAMBivBtgN0IAAACWSlgIWLhwofr27auUlBQVFhZqw4YNiZoKAACchoSEgOeff17Tp0/XnDlz9O6772rYsGEaN26cqqqqEjEdAOA08b0CdktICJg/f75uueUWTZ48WRdccIGefPJJpaWl6ZlnnknEdAAA4DQkOz1gY2OjNm/erPvvvz/W5vV6VVxcrLVr1zbrHw6HFQ6HY49DoZAkKRKJKBKJOL08AGi3uqcH9NlXDa36nezOfqX5vnk954k2nfR3coMpqgx9/e9uVtqpHwayO/t1uF4KJfif5sxUn3KDKaqub5Qkde/sP+Xf7eRL7JkNj2le316ZKfpPzdG4Np/XnPAY9t12J491rR3L8RBQXV2tpqYm5eTkxLXn5OToww8/bNa/pKREc+fObdZeWlqqtLQ0R9Z071Dpkfccf6qAtYZ2ieq9g+3juuJfDWhSXaP0r0+TlOQxajLNDwK9OxntO/z9B4fpQ45p/vbv/3ciO8Wo+ug346QkGR1tih/3u2vo09nI5zW6uJvR1q88+rjOo+FdjL4KS3tC39SwXzCqa7p8qYyIV+EmaVCm0ZMfJrW4jj8MP6ZH30tSY9SjH2ZUKvBFpYZ38ap/htHGt1aqSyBJB8Mt12FAptEl3b7Stv8ky+c1usi7T++mJ2lvnUepSUZHmk5cp+KMKjUFpX/u9WpkttH/3fP1+gZkRPVRrVfJHqNj36l/Ufeo1lbF7yuF3aJa/+U3bf+rb5P+/enXYw3MiOrHfY4okHRETQ1eFedF1avTAZX993DVPxjV7lD8eEOyouqXYfRxyKP0L3foJ309WvZpfO0uzYlqTaVXhd2iKkg3WvpJUmwdF3aNqvqoR/sPe9Srk1FNo1Qf8eiWgU1aX+WJ7esZfqMd61ZpfL5H1Uc9qjzi0af1Ht3Rr14f1Hi09SuP8tKkmkapeuc6Lf9Amj5Ymr/j67UP7RJVfiej5cuXa0iWV9sPeTWqW1RlZWUnrHlrNTS0LkR6jHH2JogHDhxQz5499c4776ioqCjWfu+996q8vFzr16+P69/SmYD8/HxVV1crGAw6sqZIJKKysjL98Ic/lM/nc2RM21FT51FTZ1FP51FTZyWinqFQSNnZ2aqtrT2lY6jjL4+zs7OVlJSkysrKuPbKykrl5uY26x8IBBQIBJq1+3w+x3eyRIxpO2rqPGrqLOrpPGrqLCfr2dpxHD+f5/f7NWLECK1cuTLWFo1GtXLlyrgzAwAAwF0JeaN8+vTpmjRpkkaOHKmLL75YCxYs0OHDhzV58uRETAcAAE5DQkLAhAkT9OWXX2r27NmqqKjQ8OHDtWLFimYXCwIAAPck7JL5qVOnaurUqYkaHgAAnKH28RkfAADQ5ggBAABYihAAAIClCAEAAFiKEAAAgKUIAQAAWIoQAACApQgBAABYihAAAIClEvaNgafr+J2NQ6GQY2NGIhE1NDQoFApx5yuHUFPnUVNnUU/nUVNnJaKex4+dx4+lJ9PuQkBdXZ0kKT8/3+WVAADQMdXV1SkjI+Ok/TzmVONCG4lGozpw4IDS09Pl8XgcGTMUCik/P1/79+9XMBh0ZEzbUVPnUVNnUU/nUVNnJaKexhjV1dUpLy9PXu/J3/Fvd2cCvF6vevXqlZCxg8EgO67DqKnzqKmzqKfzqKmznK7nqZwBOI4LAwEAsBQhAAAAS1kRAgKBgObMmaNAIOD2Us4a1NR51NRZ1NN51NRZ7aGe7e7CQAAA0DasOBMAAACaIwQAAGApQgAAAJYiBAAAYClCAAAAlrIiBCxcuFB9+/ZVSkqKCgsLtWHDBreX5IrVq1fruuuuU15enjwej1566aW47cYYzZ49Wz169FBqaqqKi4u1e/fuuD4HDx7UxIkTFQwGlZmZqV//+teqr6+P6/Pee+/psssuU0pKivLz8/XII480W8uLL76ogQMHKiUlRUOGDNHy5csdf76JVlJSolGjRik9PV3du3fXDTfcoF27dsX1OXr0qKZMmaKuXbuqc+fO+ulPf6rKysq4Pvv27dP48eOVlpam7t27a8aMGTp27Fhcn1WrVumiiy5SIBBQv379tHjx4mbr6ej7+aJFizR06NDYt6cVFRXp9ddfj22nlmdm3rx58ng8mjZtWqyNmrbO//zP/8jj8cT9DBw4MLa9Q9bTnOWWLl1q/H6/eeaZZ8z7779vbrnlFpOZmWkqKyvdXlqbW758ufnDH/5g/v3vfxtJZtmyZXHb582bZzIyMsxLL71ktm3bZn784x+bgoICc+TIkVifH/3oR2bYsGFm3bp15q233jL9+vUzN910U2x7bW2tycnJMRMnTjQ7duwwS5YsMampqeapp56K9Xn77bdNUlKSeeSRR8zOnTvNH//4R+Pz+cz27dsTXgMnjRs3zjz77LNmx44dZuvWreaaa64xvXv3NvX19bE+t99+u8nPzzcrV640mzZtMj/4wQ/MJZdcEtt+7NgxM3jwYFNcXGy2bNlili9fbrKzs839998f6/PJJ5+YtLQ0M336dLNz507zxBNPmKSkJLNixYpYn7NhP3/llVfMa6+9Zj766COza9cu8/vf/974fD6zY8cOYwy1PBMbNmwwffv2NUOHDjV33313rJ2ats6cOXPMoEGDzBdffBH7+fLLL2PbO2I9z/oQcPHFF5spU6bEHjc1NZm8vDxTUlLi4qrc990QEI1GTW5urnn00UdjbTU1NSYQCJglS5YYY4zZuXOnkWQ2btwY6/P6668bj8djPv/8c2OMMX/9619NVlaWCYfDsT733XefOe+882KPf/7zn5vx48fHraewsNDcdtttjj7HtlZVVWUkmfLycmPM1/Xz+XzmxRdfjPX54IMPjCSzdu1aY8zXwczr9ZqKiopYn0WLFplgMBir4b333msGDRoUN9eECRPMuHHjYo/P1v08KyvL/P3vf6eWZ6Curs7079/flJWVmSuuuCIWAqhp682ZM8cMGzasxW0dtZ5n9dsBjY2N2rx5s4qLi2NtXq9XxcXFWrt2rYsra3/27t2rioqKuFplZGSosLAwVqu1a9cqMzNTI0eOjPUpLi6W1+vV+vXrY30uv/xy+f3+WJ9x48Zp165dOnToUKzPt+c53qej/01qa2slSV26dJEkbd68WZFIJO65Dhw4UL17946r6ZAhQ5STkxPrM27cOIVCIb3//vuxPt9Xr7NxP29qatLSpUt1+PBhFRUVUcszMGXKFI0fP77Z86amp2f37t3Ky8vTOeeco4kTJ2rfvn2SOm49z+oQUF1draampriCS1JOTo4qKipcWlX7dLwe31eriooKde/ePW57cnKyunTpEtenpTG+PceJ+nTkv0k0GtW0adM0evRoDR48WNLXz9Pv9yszMzOu73drerr1CoVCOnLkyFm1n2/fvl2dO3dWIBDQ7bffrmXLlumCCy6glqdp6dKlevfdd1VSUtJsGzVtvcLCQi1evFgrVqzQokWLtHfvXl122WWqq6vrsPVsd7cSBjqiKVOmaMeOHVqzZo3bS+nQzjvvPG3dulW1tbX65z//qUmTJqm8vNztZXVI+/fv1913362ysjKlpKS4vZyzwtVXXx3776FDh6qwsFB9+vTRCy+8oNTUVBdXdvrO6jMB2dnZSkpKanZ1ZmVlpXJzc11aVft0vB7fV6vc3FxVVVXFbT927JgOHjwY16elMb49x4n6dNS/ydSpU/Xqq6/qzTffVK9evWLtubm5amxsVE1NTVz/79b0dOsVDAaVmpp6Vu3nfr9f/fr104gRI1RSUqJhw4bpscceo5anYfPmzaqqqtJFF12k5ORkJScnq7y8XI8//riSk5OVk5NDTc9QZmamBgwYoD179nTYffSsDgF+v18jRozQypUrY23RaFQrV65UUVGRiytrfwoKCpSbmxtXq1AopPXr18dqVVRUpJqaGm3evDnW54033lA0GlVhYWGsz+rVqxWJRGJ9ysrKdN555ykrKyvW59vzHO/T0f4mxhhNnTpVy5Yt0xtvvKGCgoK47SNGjJDP54t7rrt27dK+ffviarp9+/a4cFVWVqZgMKgLLrgg1uf76nU27+fRaFThcJhanoYxY8Zo+/bt2rp1a+xn5MiRmjhxYuy/qemZqa+v18cff6wePXp03H201ZcSdjBLly41gUDALF682OzcudPceuutJjMzM+7qTFvU1dWZLVu2mC1bthhJZv78+WbLli3ms88+M8Z8/RHBzMxM8/LLL5v33nvPXH/99S1+RPDCCy8069evN2vWrDH9+/eP+4hgTU2NycnJMb/85S/Njh07zNKlS01aWlqzjwgmJyebP//5z+aDDz4wc+bM6ZAfEbzjjjtMRkaGWbVqVdxHhhoaGmJ9br/9dtO7d2/zxhtvmE2bNpmioiJTVFQU2378I0Njx441W7duNStWrDDdunVr8SNDM2bMMB988IFZuHBhix8Z6uj7+cyZM015ebnZu3evee+998zMmTONx+MxpaWlxhhq6YRvfzrAGGraWvfcc49ZtWqV2bt3r3n77bdNcXGxyc7ONlVVVcaYjlnPsz4EGGPME088YXr37m38fr+5+OKLzbp169xekivefPNNI6nZz6RJk4wxX39McNasWSYnJ8cEAgEzZswYs2vXrrgxvvrqK3PTTTeZzp07m2AwaCZPnmzq6uri+mzbts1ceumlJhAImJ49e5p58+Y1W8sLL7xgBgwYYPx+vxk0aJB57bXXEva8E6WlWkoyzz77bKzPkSNHzJ133mmysrJMWlqa+clPfmK++OKLuHE+/fRTc/XVV5vU1FSTnZ1t7rnnHhOJROL6vPnmm2b48OHG7/ebc845J26O4zr6fn7zzTebPn36GL/fb7p162bGjBkTCwDGUEsnfDcEUNPWmTBhgunRo4fx+/2mZ8+eZsKECWbPnj2x7R2xnh5jjGn9+QMAANDRndXXBAAAgBMjBAAAYClCAAAAliIEAABgKUIAAACWIgQAAGApQgAAAJYiBAAAYClCAAAAliIEAABgKUIAAACW+v8gCFWZhlDm7AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 600x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "arms = 10\n",
    "N,D_IN,H,D_OUT = 1,arms,100,arms\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_IN,H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H,D_OUT),\n",
    "    torch.nn.ReLU(),\n",
    "    \n",
    ")\n",
    "\n",
    "def one_hot(arms,pos,val=1):\n",
    "    one_hot_vec = np.zeros(arms)\n",
    "    if val != 0:\n",
    "        one_hot_vec[pos]= val\n",
    "    return one_hot_vec\n",
    "\n",
    "\n",
    "def train(env,model,epochs=50000):\n",
    "    # loss function\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(),lr=0.0001)\n",
    "\n",
    "    # logging\n",
    "    cur_state = torch.tensor(one_hot(arms,env.get_state()),dtype=torch.float32) # 获得当前的状态\n",
    "    # 上下文是怎样体现的？\n",
    "\n",
    "    rewards = []\n",
    "\n",
    "    for i in range(epochs):\n",
    "        y_pred = model(cur_state) # 预测的是return 输如state 输出对应的return \n",
    "        av_softmax = softmax(y_pred.data.numpy())\n",
    "        av_softmax /= av_softmax.sum()\n",
    "\n",
    "        choice = np.random.choice(arms,p=av_softmax) # 随机选择新的动作\n",
    "        cur_reward = env.choose_arm(choice) # 获得奖励\n",
    "\n",
    "        one_hot_reward = y_pred.data.numpy().copy()\n",
    "        one_hot_reward[choice] = cur_reward\n",
    "\n",
    "        reward = torch.tensor(one_hot_reward)\n",
    "        rewards.append(cur_reward)\n",
    "        \n",
    "        loss = loss_fn(y_pred,reward)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        cur_state = torch.tensor(one_hot(arms,env.get_state()),dtype=torch.float32)\n",
    "\n",
    "    return rewards\n",
    "\n",
    "env = ContextualBandit(10)\n",
    "state = env.get_state()\n",
    "reward  = env.choose_arm(1)\n",
    "rewards = train(env,model)\n",
    "\n",
    "plot_result(rewards)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "procthor-rl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
